From guido at codespeak.net Sat Aug 5 00:18:44 2006 From: guido at codespeak.net (guido at codespeak.net) Date: Sat, 5 Aug 2006 00:18:44 +0200 (CEST) Subject: [py-svn] r31014 - in py/dist/py/test: . testing Message-ID: <20060804221844.8C37F1006E@code0.codespeak.net> Author: guido Date: Sat Aug 5 00:18:33 2006 New Revision: 31014 Modified: py/dist/py/test/config.py py/dist/py/test/testing/test_config.py Log: Allowing options with a 'dest' attribute (used in PyPy). Modified: py/dist/py/test/config.py ============================================================================== --- py/dist/py/test/config.py (original) +++ py/dist/py/test/config.py Sat Aug 5 00:18:33 2006 @@ -79,7 +79,8 @@ optgroup.add_options(specs) parser.add_option_group(optgroup) for opt in specs: - if hasattr(opt, 'default'): + if hasattr(opt, 'default'): + print 'foo' setattr(cls._config.option, opt.dest, opt.default) return cls._config.option addoptions = classmethod(addoptions) Modified: py/dist/py/test/testing/test_config.py ============================================================================== --- py/dist/py/test/testing/test_config.py (original) +++ py/dist/py/test/testing/test_config.py Sat Aug 5 00:18:33 2006 @@ -32,10 +32,15 @@ o = py.test.ensuretemp('configoptions') o.ensure("conftest.py").write(py.code.Source(""" import py + def _callback(option, opt_str, value, parser, *args, **kwargs): + option.tdest = True Option = py.test.Config.Option option = py.test.Config.addoptions("testing group", Option('-g', '--glong', action="store", default=42, type="int", dest="gdest", help="g value."), + # XXX note: special case, option without a destination + Option('-t', '--tlong', action="callback", callback=_callback, + help='t value'), ) """)) old = o.chdir() From guido at codespeak.net Sat Aug 5 00:21:31 2006 From: guido at codespeak.net (guido at codespeak.net) Date: Sat, 5 Aug 2006 00:21:31 +0200 (CEST) Subject: [py-svn] r31015 - py/dist/py/test Message-ID: <20060804222131.013E710074@code0.codespeak.net> Author: guido Date: Sat Aug 5 00:21:31 2006 New Revision: 31015 Modified: py/dist/py/test/config.py Log: Oops, had checked in debug print instead of the fix :( Modified: py/dist/py/test/config.py ============================================================================== --- py/dist/py/test/config.py (original) +++ py/dist/py/test/config.py Sat Aug 5 00:21:31 2006 @@ -79,8 +79,7 @@ optgroup.add_options(specs) parser.add_option_group(optgroup) for opt in specs: - if hasattr(opt, 'default'): - print 'foo' + if hasattr(opt, 'default') and opt.dest: setattr(cls._config.option, opt.dest, opt.default) return cls._config.option addoptions = classmethod(addoptions) From hpk at codespeak.net Thu Aug 10 07:05:13 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 10 Aug 2006 07:05:13 +0200 (CEST) Subject: [py-svn] r31212 - py/dist/py/documentation Message-ID: <20060810050513.D67911005A@code0.codespeak.net> Author: hpk Date: Thu Aug 10 07:05:11 2006 New Revision: 31212 Modified: py/dist/py/documentation/execnet.txt Log: fix typo (thanks Ondrei) Modified: py/dist/py/documentation/execnet.txt ============================================================================== --- py/dist/py/documentation/execnet.txt (original) +++ py/dist/py/documentation/execnet.txt Thu Aug 10 07:05:11 2006 @@ -136,7 +136,7 @@ import py contentserverbootstrap = py.code.Source( """ - for fn in channel.receive(): + for fn in channel: f = open(fn, 'rb') try: channel.send(f.read()) From guido at codespeak.net Thu Aug 10 17:59:13 2006 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 10 Aug 2006 17:59:13 +0200 (CEST) Subject: [py-svn] r31236 - in py/dist/py/xmlobj: . testing Message-ID: <20060810155913.02DB810063@code0.codespeak.net> Author: guido Date: Thu Aug 10 17:59:12 2006 New Revision: 31236 Modified: py/dist/py/xmlobj/html.py py/dist/py/xmlobj/testing/test_html.py py/dist/py/xmlobj/visit.py Log: (jan, guido) Fixed problem in HTML generation, where certain tags must be closed using an explicit closing tag, and others must be a singleton. Modified: py/dist/py/xmlobj/html.py ============================================================================== --- py/dist/py/xmlobj/html.py (original) +++ py/dist/py/xmlobj/html.py Thu Aug 10 17:59:12 2006 @@ -6,6 +6,9 @@ from py.__.xmlobj.visit import SimpleUnicodeVisitor class HtmlVisitor(SimpleUnicodeVisitor): + + singletons = ['area', 'link'] + def repr_attribute(self, attrs, name): if name == 'class_': value = getattr(attrs, name) @@ -13,6 +16,9 @@ return return super(HtmlVisitor, self).repr_attribute(attrs, name) + def _issingleton(self, tagname): + return tagname in self.singletons + class HtmlTag(Tag): def unicode(self, indent=2): l = [] Modified: py/dist/py/xmlobj/testing/test_html.py ============================================================================== --- py/dist/py/xmlobj/testing/test_html.py (original) +++ py/dist/py/xmlobj/testing/test_html.py Thu Aug 10 17:59:12 2006 @@ -39,3 +39,10 @@ '
  • world
    • ' '
  • 42
    • ' '') + +def test_singleton(): + h = html.head(html.link(href="foo")) + assert unicode(h) == '' + + h = html.head(html.script(src="foo")) + assert unicode(h) == '' Modified: py/dist/py/xmlobj/visit.py ============================================================================== --- py/dist/py/xmlobj/visit.py (original) +++ py/dist/py/xmlobj/visit.py Thu Aug 10 17:59:12 2006 @@ -59,7 +59,7 @@ self.curindent -= self.indent else: nameattr = tagname+self.attributes(tag) - if self.shortempty: + if self._issingleton(tagname): self.write(u'<%s/>' % (nameattr,)) else: self.write(u'<%s>' % (nameattr, tagname)) @@ -93,4 +93,7 @@ stylelist = [x+': ' + y for x,y in styledict.items()] return [u' style="%s"' % u'; '.join(stylelist)] + def _issingleton(self, tagname): + """can (and will) be overridden in subclasses""" + return self.shortempty From guido at codespeak.net Thu Aug 10 18:17:07 2006 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 10 Aug 2006 18:17:07 +0200 (CEST) Subject: [py-svn] r31237 - py/dist/py/xmlobj Message-ID: <20060810161707.961D310063@code0.codespeak.net> Author: guido Date: Thu Aug 10 18:17:05 2006 New Revision: 31237 Modified: py/dist/py/xmlobj/html.py Log: (jan, guido) Completed lists of HTML tags. Modified: py/dist/py/xmlobj/html.py ============================================================================== --- py/dist/py/xmlobj/html.py (original) +++ py/dist/py/xmlobj/html.py Thu Aug 10 18:17:05 2006 @@ -7,7 +7,24 @@ class HtmlVisitor(SimpleUnicodeVisitor): - singletons = ['area', 'link'] + not_single = ['a', 'abbr', 'acronym', 'address', 'applet', + 'b', 'bdo', 'big', 'blink', 'blockquote', + 'body', 'button', 'caption', 'center', 'cite', + 'comment', 'del', 'dfn', 'dir', 'div', + 'dl', 'dt', 'em', 'embed', 'fieldset', + 'font', 'form', 'frameset', 'h1', 'h2', + 'h3', 'h4', 'h5', 'h6', 'head', 'html', + 'i', 'iframe', 'ins', 'kbd', 'label', 'legend', + 'li', 'listing', 'map', 'marquee', 'menu', + 'multicol', 'nobr', 'noembed', 'noframes', + 'noscript', 'object', 'ol', 'optgroup', + 'option', 'p', 'pre', 'q', 's', 'script', + 'select', 'small', 'span', 'strike', + 'strong', 'style', 'sub', 'sup', 'table', + 'tbody', 'td', 'textarea', 'tfoot', + 'th', 'thead', 'title', 'tr', 'tt', 'u', + 'ul', 'xmp'] + def repr_attribute(self, attrs, name): if name == 'class_': @@ -17,7 +34,7 @@ return super(HtmlVisitor, self).repr_attribute(attrs, name) def _issingleton(self, tagname): - return tagname in self.singletons + return tagname not in self.not_single class HtmlTag(Tag): def unicode(self, indent=2): @@ -30,12 +47,16 @@ __tagclass__ = HtmlTag __stickyname__ = True __tagspec__ = dict([(x,1) for x in ( - "h1,h2,h3,h5,h6,p,b,i,a,div,span,code," - "html,head,title,style,table,thead,tr,tt," - "td,th,link,img,meta,body,pre,br,ul," - "ol,li,em,form,input,select,option," - "button,script,colgroup,col,map,area," - "blockquote,dl,dt,dd,strong" + 'a,abbr,acronym,address,applet,area,b,bdo,big,blink,' + 'blockquote,body,br,button,caption,center,cite,code,col,' + 'colgroup,comment,dd,del,dfn,dir,div,dl,dt,em,embed,' + 'fieldset,font,form,frameset,h1,h2,h3,h4,h5,h6,head,html,' + 'i,iframe,img,input,ins,kbd,label,legend,li,link,listing,' + 'map,marquee,menu,meta,multicol,nobr,noembed,noframes,' + 'noscript,object,ol,optgroup,option,p,pre,q,s,script,' + 'select,small,span,strike,strong,style,sub,sup,table,' + 'tbody,td,textarea,tfoot,th,thead,title,tr,tt,u,ul,xmp,' + 'base,basefont,frame,hr,isindex,param,samp,var' ).split(',') if x]) class Style(object): From guido at codespeak.net Thu Aug 10 18:52:37 2006 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 10 Aug 2006 18:52:37 +0200 (CEST) Subject: [py-svn] r31238 - py/dist/py/xmlobj Message-ID: <20060810165237.8A4AD10063@code0.codespeak.net> Author: guido Date: Thu Aug 10 18:52:36 2006 New Revision: 31238 Modified: py/dist/py/xmlobj/html.py Log: Defined 'single' rather than 'not_single' to check how a tag must be closed (the list of singletons is much smaller than the list of tags that must have a closing tag). Modified: py/dist/py/xmlobj/html.py ============================================================================== --- py/dist/py/xmlobj/html.py (original) +++ py/dist/py/xmlobj/html.py Thu Aug 10 18:52:36 2006 @@ -7,24 +7,9 @@ class HtmlVisitor(SimpleUnicodeVisitor): - not_single = ['a', 'abbr', 'acronym', 'address', 'applet', - 'b', 'bdo', 'big', 'blink', 'blockquote', - 'body', 'button', 'caption', 'center', 'cite', - 'comment', 'del', 'dfn', 'dir', 'div', - 'dl', 'dt', 'em', 'embed', 'fieldset', - 'font', 'form', 'frameset', 'h1', 'h2', - 'h3', 'h4', 'h5', 'h6', 'head', 'html', - 'i', 'iframe', 'ins', 'kbd', 'label', 'legend', - 'li', 'listing', 'map', 'marquee', 'menu', - 'multicol', 'nobr', 'noembed', 'noframes', - 'noscript', 'object', 'ol', 'optgroup', - 'option', 'p', 'pre', 'q', 's', 'script', - 'select', 'small', 'span', 'strike', - 'strong', 'style', 'sub', 'sup', 'table', - 'tbody', 'td', 'textarea', 'tfoot', - 'th', 'thead', 'title', 'tr', 'tt', 'u', - 'ul', 'xmp'] - + single = dict([(x, 1) for x in + ('br,img,area,param,col,hr,meta,link,base,' + 'input,frame').split(',')]) def repr_attribute(self, attrs, name): if name == 'class_': @@ -34,7 +19,7 @@ return super(HtmlVisitor, self).repr_attribute(attrs, name) def _issingleton(self, tagname): - return tagname not in self.not_single + return tagname in self.single class HtmlTag(Tag): def unicode(self, indent=2): From hpk at codespeak.net Mon Aug 21 11:48:28 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 21 Aug 2006 11:48:28 +0200 (CEST) Subject: [py-svn] r31432 - py/branch/distributed Message-ID: <20060821094828.956C810074@code0.codespeak.net> Author: hpk Date: Mon Aug 21 11:48:26 2006 New Revision: 31432 Added: py/branch/distributed/ - copied from r31431, py/dist/ Log: branching off for distributed testing impl. From hpk at codespeak.net Mon Aug 21 13:29:19 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 21 Aug 2006 13:29:19 +0200 (CEST) Subject: [py-svn] r31437 - in py/branch/distributed/py/test: . tkinter Message-ID: <20060821112919.D134310078@code0.codespeak.net> Author: hpk Date: Mon Aug 21 13:29:17 2006 New Revision: 31437 Modified: py/branch/distributed/py/test/session.py py/branch/distributed/py/test/tkinter/reportsession.py Log: warning doesn't appear to be used Modified: py/branch/distributed/py/test/session.py ============================================================================== --- py/branch/distributed/py/test/session.py (original) +++ py/branch/distributed/py/test/session.py Mon Aug 21 13:29:17 2006 @@ -35,9 +35,6 @@ def getitemoutcomepairs(self, cls): return [x for x in self._memo if isinstance(x[1], cls)] - def warning(self, msg): - raise Warning(msg) - def main(self, args): """ main loop for running tests. """ colitems = self._map2colitems(args) Modified: py/branch/distributed/py/test/tkinter/reportsession.py ============================================================================== --- py/branch/distributed/py/test/tkinter/reportsession.py (original) +++ py/branch/distributed/py/test/tkinter/reportsession.py Mon Aug 21 13:29:17 2006 @@ -43,6 +43,4 @@ def sendreport(self, report): self.channel.send(report.to_channel()) - def warning(self, msg): - pass From hpk at codespeak.net Mon Aug 21 19:12:35 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 21 Aug 2006 19:12:35 +0200 (CEST) Subject: [py-svn] r31449 - in py/branch/distributed/py: . test/rsession test/terminal Message-ID: <20060821171235.386C010075@code0.codespeak.net> Author: hpk Date: Mon Aug 21 19:12:30 2006 New Revision: 31449 Added: py/branch/distributed/py/test/rsession/ py/branch/distributed/py/test/rsession/__init__.py (contents, props changed) py/branch/distributed/py/test/rsession/rsession.py - copied, changed from r31437, py/branch/distributed/py/test/session.py Modified: py/branch/distributed/py/__init__.py py/branch/distributed/py/test/terminal/out.py Log: intermediate checkin Modified: py/branch/distributed/py/__init__.py ============================================================================== --- py/branch/distributed/py/__init__.py (original) +++ py/branch/distributed/py/__init__.py Mon Aug 21 19:12:30 2006 @@ -37,6 +37,7 @@ # for customization of collecting/running tests 'test.Session' : ('./test/session.py', 'Session'), 'test.TerminalSession' : ('./test/terminal/terminal.py', 'TerminalSession'), + 'test.RSession' : ('./test/rsession/rsession.py', 'RSession'), 'test.TkinterSession' : ('./test/tkinter/tksession.py', 'TkSession'), 'test.collect.Collector' : ('./test/collect.py', 'Collector'), 'test.collect.Directory' : ('./test/collect.py', 'Directory'), Added: py/branch/distributed/py/test/rsession/__init__.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/__init__.py Mon Aug 21 19:12:30 2006 @@ -0,0 +1 @@ +# Copied: py/branch/distributed/py/test/rsession/rsession.py (from r31437, py/branch/distributed/py/test/session.py) ============================================================================== --- py/branch/distributed/py/test/session.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Mon Aug 21 19:12:30 2006 @@ -1,73 +1,48 @@ import py -class Session(object): - """ - A Session gets test Items from Collectors, # executes the - Items and sends the Outcome to the Reporter. - """ - def __init__(self, config): - self._memo = [] - self.config = config +class Reporter(object): + def __init__(self, config): + self.config = config - def shouldclose(self): - return False + def start(self, colitem): + print "starting colitem", colitem - def header(self, colitems): - """ setup any neccessary resources. """ - if not self.config.option.nomagic: - py.magic.invoke(assertion=1) - - def footer(self, colitems): - """ teardown any resources we know about. """ - py.test.Function.state.teardown_all() - if not self.config.option.nomagic: - py.magic.revoke(assertion=1) - - def start(self, colitem): - pass + def finish(self, colitem, outcome): + print "finishing colitem", colitem, "outcome", outcome - def finish(self, colitem, outcome): - self._memo.append((colitem, outcome)) - def startiteration(self, colitem, subitems): - pass + +class RSession(object): + """ + An RSession executes collectors/items through a runner. - def getitemoutcomepairs(self, cls): - return [x for x in self._memo if isinstance(x[1], cls)] + """ + def __init__(self, config): + self.config = config + self.report = Reporter(config) def main(self, args): """ main loop for running tests. """ colitems = self._map2colitems(args) - try: - self.header(colitems) - try: - for colitem in colitems: - colitem.option = self.config.option - self.runtraced(colitem) - except KeyboardInterrupt: - raise - except: - self.footer(colitems) - else: - self.footer(colitems) - except Exit, ex: - pass - # return [(fspath as string, [names as string])] - return [(str(item.listchain()[0].fspath), item.listnames()) - for item, outcome in self.getitemoutcomepairs(py.test.Item.Failed)] + print "hello world" + print colitems + raise SystemExit, 1 + count = 0 + while colitems: + colitem = colitems.pop() + self.runtraced(colitem) + if len(self.results) < count: + time.sleep(0.1) def runtraced(self, colitem): - if self.shouldclose(): - raise Exit, "received external close signal" - outcome = None colitem.startcapture() try: - self.start(colitem) + self.report.predispatch(colitem) + self.dispatch(colitem) + self.report.postdispatch(colitem) try: try: - if colitem._stickyfailure: - raise colitem._stickyfailure outcome = self.run(colitem) except (KeyboardInterrupt, Exit): raise Modified: py/branch/distributed/py/test/terminal/out.py ============================================================================== --- py/branch/distributed/py/test/terminal/out.py (original) +++ py/branch/distributed/py/test/terminal/out.py Mon Aug 21 19:12:30 2006 @@ -80,7 +80,7 @@ # if file is None: file = py.std.sys.stdout - elif hasattr(file, 'send'): + elif hasattr(file, 'send'): # likely a channel like thing file = WriteFile(file.send) if hasattr(file, 'isatty') and file.isatty(): return TerminalOut(file) From hpk at codespeak.net Mon Aug 21 20:54:30 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 21 Aug 2006 20:54:30 +0200 (CEST) Subject: [py-svn] r31453 - py/branch/distributed Message-ID: <20060821185430.8B63B10079@code0.codespeak.net> Author: hpk Date: Mon Aug 21 20:54:28 2006 New Revision: 31453 Added: py/branch/distributed/TODO Log: TODO for 1st milestone, discussed by fijal, hpk and a bit of armin Added: py/branch/distributed/TODO ============================================================================== --- (empty file) +++ py/branch/distributed/TODO Mon Aug 21 20:54:28 2006 @@ -0,0 +1,42 @@ +goal: interactive distributed testing with a given +set of nodes (SSH-hosts) + +1st. Milestone +----------------- + +* use "/usr/bin/rsync" to sync to remote +* master does collection of items +* "leaf" items are asynchronously send to "Executors" (running on nodes) +* super-simple reporting + +# running at MASTER: +channels = [] +for gw in mygateways: + c = gw.remote_exec(executor_protocol) + c.setcallback(lambda x: report_queue.put) + channels.append(c) + +try: + # wait for completion +finally: + for c in channels: + c.send(None) + c.close() + +# dispatching thread: +while 1: + for node in nodes: + if len(node.pending) < MAXTASKS_PER_NODE: + item = itemgenerator.next() + if shouldstop: + break + node.send(item) + + +# reporting thread +while 1: + if shouldstop: + break + report_event = report_queue.get() + print report_event + From hpk at codespeak.net Tue Aug 22 10:57:40 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 10:57:40 +0200 (CEST) Subject: [py-svn] r31456 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060822085740.653D51005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 10:57:38 2006 New Revision: 31456 Added: py/branch/distributed/py/test/rsession/testing/ Log: add testing directory From hpk at codespeak.net Tue Aug 22 11:47:46 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 11:47:46 +0200 (CEST) Subject: [py-svn] r31457 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060822094746.3AF8B10060@code0.codespeak.net> Author: hpk Date: Tue Aug 22 11:47:42 2006 New Revision: 31457 Added: py/branch/distributed/py/test/rsession/testing/__init__.py (contents, props changed) py/branch/distributed/py/test/rsession/testing/example1.py (contents, props changed) py/branch/distributed/py/test/rsession/testing/test_executor.py (contents, props changed) Log: intermediate Added: py/branch/distributed/py/test/rsession/testing/__init__.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/__init__.py Tue Aug 22 11:47:42 2006 @@ -0,0 +1 @@ +# Added: py/branch/distributed/py/test/rsession/testing/example1.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/example1.py Tue Aug 22 11:47:42 2006 @@ -0,0 +1,13 @@ + + +def f1(): + f2() + +def f2(): + pass + +def g1(): + g2() + +def g2(): + assert ValueError Added: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Tue Aug 22 11:47:42 2006 @@ -0,0 +1,16 @@ + +import py +import example1 + +from py.__.test.rsession.executor import Executor + +def test_executor_passing_function(): + ex = Executor(example1.f1) + outcome = ex.execute() + assert outcome.haspassed() + +def test_executor_raising_function(): + ex = Executor(example1.g1) + outcome = ex.execute() + assert not outcome.haspassed() + excinfo = outcome.getexcinfo() From fijal at codespeak.net Tue Aug 22 12:00:42 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 12:00:42 +0200 (CEST) Subject: [py-svn] r31458 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822100042.0143B1005A@code0.codespeak.net> Author: fijal Date: Tue Aug 22 12:00:40 2006 New Revision: 31458 Added: py/branch/distributed/py/test/rsession/executor.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/testing/example1.py py/branch/distributed/py/test/rsession/testing/test_executor.py Log: Added simple executor. Added: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/executor.py Tue Aug 22 12:00:40 2006 @@ -0,0 +1,32 @@ +""" Remote executor +""" + +import sys +import py + +class Outcome(object): + pass + +class FailingOutcome(Outcome): + def __init__(self, excinfo): + self.excinfo = excinfo + + def haspassed(self): + return False + +class PassingOutcome(Outcome): + def haspassed(self): + return True + +class Executor(object): + def __init__(self, fun): + self.fun = fun + + def execute(self): + try: + self.fun() + except Exception, e: + excinfo = py.code.ExceptionInfo() + return FailingOutcome(excinfo) + else: + return PassingOutcome() Modified: py/branch/distributed/py/test/rsession/testing/example1.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/example1.py (original) +++ py/branch/distributed/py/test/rsession/testing/example1.py Tue Aug 22 12:00:40 2006 @@ -10,4 +10,4 @@ g2() def g2(): - assert ValueError + raise ValueError() Modified: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_executor.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Tue Aug 22 12:00:40 2006 @@ -9,8 +9,9 @@ outcome = ex.execute() assert outcome.haspassed() -def test_executor_raising_function(): +def test_executor_raising_function(): ex = Executor(example1.g1) outcome = ex.execute() assert not outcome.haspassed() - excinfo = outcome.getexcinfo() + excinfo = outcome.excinfo + assert excinfo.type == ValueError From fijal at codespeak.net Tue Aug 22 12:02:05 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 12:02:05 +0200 (CEST) Subject: [py-svn] r31459 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822100205.D063E1005A@code0.codespeak.net> Author: fijal Date: Tue Aug 22 12:02:04 2006 New Revision: 31459 Modified: py/branch/distributed/py/test/rsession/ (props changed) py/branch/distributed/py/test/rsession/testing/ (props changed) Log: fixed eol and ignore prop From fijal at codespeak.net Tue Aug 22 12:53:16 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 12:53:16 +0200 (CEST) Subject: [py-svn] r31462 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822105316.BE10010060@code0.codespeak.net> Author: fijal Date: Tue Aug 22 12:53:13 2006 New Revision: 31462 Modified: py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/testing/example1.py py/branch/distributed/py/test/rsession/testing/test_executor.py Log: Added traceback test. Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Tue Aug 22 12:53:13 2006 @@ -27,6 +27,8 @@ self.fun() except Exception, e: excinfo = py.code.ExceptionInfo() + code = py.code.Code(self.fun) + excinfo.traceback = excinfo.traceback.cut(path=code.path, firstlineno=code.firstlineno) return FailingOutcome(excinfo) else: return PassingOutcome() Modified: py/branch/distributed/py/test/rsession/testing/example1.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/example1.py (original) +++ py/branch/distributed/py/test/rsession/testing/example1.py Tue Aug 22 12:53:13 2006 @@ -1,5 +1,4 @@ - def f1(): f2() Modified: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_executor.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Tue Aug 22 12:53:13 2006 @@ -15,3 +15,11 @@ assert not outcome.haspassed() excinfo = outcome.excinfo assert excinfo.type == ValueError + +def test_executor_traceback(): + ex = Executor(example1.g1) + outcome = ex.execute() + excinfo = outcome.excinfo + assert len(excinfo.traceback) == 2 + assert excinfo.traceback[1].frame.code.name == 'g2' + assert excinfo.traceback[0].frame.code.name == 'g1' From hpk at codespeak.net Tue Aug 22 13:16:04 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 13:16:04 +0200 (CEST) Subject: [py-svn] r31464 - py/branch/distributed/py/test/rsession Message-ID: <20060822111604.1160210060@code0.codespeak.net> Author: hpk Date: Tue Aug 22 13:16:02 2006 New Revision: 31464 Modified: py/branch/distributed/py/test/rsession/executor.py Log: typo Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Tue Aug 22 13:16:02 2006 @@ -28,7 +28,8 @@ except Exception, e: excinfo = py.code.ExceptionInfo() code = py.code.Code(self.fun) - excinfo.traceback = excinfo.traceback.cut(path=code.path, firstlineno=code.firstlineno) + excinfo.traceback = excinfo.traceback.cut( + path=code.path, firstlineno=code.firstlineno) return FailingOutcome(excinfo) else: return PassingOutcome() From hpk at codespeak.net Tue Aug 22 13:22:47 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 13:22:47 +0200 (CEST) Subject: [py-svn] r31465 - py/branch/distributed/py/test/rsession Message-ID: <20060822112247.DD42E10060@code0.codespeak.net> Author: hpk Date: Tue Aug 22 13:22:46 2006 New Revision: 31465 Modified: py/branch/distributed/py/test/rsession/executor.py Log: removing unncessary import Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Tue Aug 22 13:22:46 2006 @@ -1,7 +1,6 @@ """ Remote executor """ -import sys import py class Outcome(object): From hpk at codespeak.net Tue Aug 22 15:06:24 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 15:06:24 +0200 (CEST) Subject: [py-svn] r31466 - in py/branch/distributed/py/test: . rsession rsession/testing Message-ID: <20060822130624.F22461005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 15:06:21 2006 New Revision: 31466 Added: py/branch/distributed/py/test/rsession/slavenode.py (contents, props changed) py/branch/distributed/py/test/rsession/testing/test_slavenode.py (contents, props changed) Modified: py/branch/distributed/py/test/collect.py Log: (fijal, hpk) make initial slavenode code work. Modified: py/branch/distributed/py/test/collect.py ============================================================================== --- py/branch/distributed/py/test/collect.py (original) +++ py/branch/distributed/py/test/collect.py Tue Aug 22 15:06:21 2006 @@ -275,21 +275,34 @@ if name in seen: continue seen[name] = True - if self.classnamefilter(name) and isclass(obj): - d[name] = self.Class(name, parent=self) - elif self.funcnamefilter(name) and callable(obj): - if obj.func_code.co_flags & 32: # generator function - d[name] = self.Generator(name, parent=self) - else: - d[name] = self.Function(name, parent=self) + res = self.makeitem(name, obj) + if res is not None: + d[name] = res return d + def makeitem(self, name, obj, usefilters=True): + if (not usefilters or self.classnamefilter(name)) and isclass(obj): + return self.Class(name, parent=self) + elif (not usefilters or self.funcnamefilter(name)) and callable(obj): + if obj.func_code.co_flags & 32: # generator function + return self.Generator(name, parent=self) + else: + return self.Function(name, parent=self) + class Module(PyCollectorMixin, FSCollector): def run(self): if getattr(self.obj, 'disabled', 0): return [] return FSCollector.run(self) + + def join(self, name): + res = super(Module, self).join(name) + if res is None: + attr = getattr(self.obj, name, None) + if attr is not None: + res = self.makeitem(name, attr, usefilters=False) + return res def startcapture(self): if not self.option.nocapture and not self.option.usepdb: Added: py/branch/distributed/py/test/rsession/slavenode.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/slavenode.py Tue Aug 22 15:06:21 2006 @@ -0,0 +1,18 @@ +""" +Node code for slaves. +""" + +class SlaveNode(object): + def __init__(self, rootcollector): + self.rootcollector = rootcollector + + def getitem(self, namelist): + if isinstance(namelist, str): + namelist = namelist.split("/") + cur = self.rootcollector + for name in namelist: + print name + next = cur.join(name) + assert next is not None, (cur, name) + cur = next + return cur Added: py/branch/distributed/py/test/rsession/testing/test_slavenode.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_slavenode.py Tue Aug 22 15:06:21 2006 @@ -0,0 +1,40 @@ + +""" Testing the slave side node code (in a local way). """ +from py.__.test.rsession.slavenode import SlaveNode +import py + +def funcpass(): + pass + +def funcfail(): + raise AssertionError("hello world") + +def getrootcol(): + rootdir = py.path.local(py.__file__).dirpath().dirpath() + rootcol = py.test.collect.Directory(rootdir) + return rootcol + +def test_slavenode_getitem(): + rootcol = getrootcol() + node = SlaveNode(rootcol) + item = node.getitem("py/test/rsession/testing/test_slavenode.py/funcpass") + assert isinstance(item, py.test.Item) + assert item.name == 'funcpass' + res = item.run() + assert res is None + +#def test_slavenode_dispatch_passing(self): + #outcome = node.run("execitem", + # "py.__.test.rsession.testing.test_slavenode.funcpass" + # .split('.'), + #) + #assert outcome.haspassed() + +#def test_slavenode_dispatch_failing(self): +# node = SlaveNode() +# outcome = node.run("execitem", +# "py.__.test.rsession.testing.test_slavenode.funcpass" +# .split('.'), +# ) +# assert not outcome.haspassed() +# assert outcome.excinfo.type == AssertionError From hpk at codespeak.net Tue Aug 22 15:23:50 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 15:23:50 +0200 (CEST) Subject: [py-svn] r31467 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822132350.533191005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 15:23:47 2006 New Revision: 31467 Modified: py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/testing/test_executor.py py/branch/distributed/py/test/rsession/testing/test_slavenode.py Log: (fijal, hpk) refine outcome interface and add setup/actual test run distinction. Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Tue Aug 22 15:23:47 2006 @@ -7,28 +7,35 @@ pass class FailingOutcome(Outcome): - def __init__(self, excinfo): + passed = False + def __init__(self, excinfo, setupfailure=False): self.excinfo = excinfo - - def haspassed(self): - return False + self.setupfailure = setupfailure class PassingOutcome(Outcome): - def haspassed(self): - return True + passed = True + setupfailure = False class Executor(object): - def __init__(self, fun): + def __init__(self, fun, setup=lambda: None): self.fun = fun + self.setup = setup - def execute(self): + def _execute(self, fun): try: - self.fun() + fun() except Exception, e: excinfo = py.code.ExceptionInfo() - code = py.code.Code(self.fun) + code = py.code.Code(fun) excinfo.traceback = excinfo.traceback.cut( path=code.path, firstlineno=code.firstlineno) - return FailingOutcome(excinfo) - else: - return PassingOutcome() + return excinfo + + def execute(self): + excinfo = self._execute(self.setup) + if excinfo is not None: + return FailingOutcome(excinfo, setupfailure=True) + excinfo = self._execute(self.fun) + if excinfo is not None: + return FailingOutcome(excinfo, setupfailure=False) + return PassingOutcome() Modified: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_executor.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Tue Aug 22 15:23:47 2006 @@ -7,12 +7,12 @@ def test_executor_passing_function(): ex = Executor(example1.f1) outcome = ex.execute() - assert outcome.haspassed() + assert outcome.passed def test_executor_raising_function(): ex = Executor(example1.g1) outcome = ex.execute() - assert not outcome.haspassed() + assert not outcome.passed excinfo = outcome.excinfo assert excinfo.type == ValueError @@ -23,3 +23,18 @@ assert len(excinfo.traceback) == 2 assert excinfo.traceback[1].frame.code.name == 'g2' assert excinfo.traceback[0].frame.code.name == 'g1' + +def test_executor_setup_passing(): + ex = Executor(example1.f1, setup=lambda: None) + outcome = ex.execute() + assert outcome.passed + assert not outcome.setupfailure + +def test_executor_setup_failing(): + def failingsetup(): + raise ValueError + ex = Executor(example1.f1, setup=failingsetup) + outcome = ex.execute() + assert not outcome.passed + assert outcome.setupfailure + assert outcome.excinfo.traceback[-1].frame.code.name == 'failingsetup' Modified: py/branch/distributed/py/test/rsession/testing/test_slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slavenode.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slavenode.py Tue Aug 22 15:23:47 2006 @@ -24,10 +24,10 @@ assert res is None #def test_slavenode_dispatch_passing(self): - #outcome = node.run("execitem", - # "py.__.test.rsession.testing.test_slavenode.funcpass" - # .split('.'), - #) +# outcome = node.run("execitem", +# "py.__.test.rsession.testing.test_slavenode.funcpass" +# .split('.'), +## ) #assert outcome.haspassed() #def test_slavenode_dispatch_failing(self): From hpk at codespeak.net Tue Aug 22 15:32:07 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 15:32:07 +0200 (CEST) Subject: [py-svn] r31469 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822133207.9CF091005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 15:32:05 2006 New Revision: 31469 Modified: py/branch/distributed/py/test/rsession/slavenode.py py/branch/distributed/py/test/rsession/testing/test_slavenode.py Log: (fijal, hpk) added run method to slavenode. Modified: py/branch/distributed/py/test/rsession/slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/slavenode.py (original) +++ py/branch/distributed/py/test/rsession/slavenode.py Tue Aug 22 15:32:05 2006 @@ -2,6 +2,8 @@ Node code for slaves. """ +from py.__.test.rsession.executor import Executor + class SlaveNode(object): def __init__(self, rootcollector): self.rootcollector = rootcollector @@ -11,8 +13,16 @@ namelist = namelist.split("/") cur = self.rootcollector for name in namelist: - print name next = cur.join(name) assert next is not None, (cur, name) cur = next return cur + + def run(self, cmd, *args): + call = getattr(self, "run_" + cmd) + return call(*args) + + def run_execitem(self, itemspec): + item = self.getitem(itemspec) + ex = Executor(item.obj, setup=item.setup) + return ex.execute() Modified: py/branch/distributed/py/test/rsession/testing/test_slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slavenode.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slavenode.py Tue Aug 22 15:32:05 2006 @@ -9,26 +9,35 @@ def funcfail(): raise AssertionError("hello world") -def getrootcol(): +def gettestnode(): rootdir = py.path.local(py.__file__).dirpath().dirpath() rootcol = py.test.collect.Directory(rootdir) - return rootcol + node = SlaveNode(rootcol) + return node def test_slavenode_getitem(): - rootcol = getrootcol() - node = SlaveNode(rootcol) + node = gettestnode() item = node.getitem("py/test/rsession/testing/test_slavenode.py/funcpass") assert isinstance(item, py.test.Item) assert item.name == 'funcpass' res = item.run() assert res is None -#def test_slavenode_dispatch_passing(self): -# outcome = node.run("execitem", -# "py.__.test.rsession.testing.test_slavenode.funcpass" -# .split('.'), -## ) - #assert outcome.haspassed() +def test_slavenode_run_passing(): + node = gettestnode() + outcome = node.run("execitem", + "py/test/rsession/testing/test_slavenode.py/funcpass") + assert outcome.passed + assert not outcome.setupfailure + +def test_slavenode_run_failing(): + node = gettestnode() + outcome = node.run("execitem", + "py/test/rsession/testing/test_slavenode.py/funcfail") + assert not outcome.passed + assert not outcome.setupfailure + assert len(outcome.excinfo.traceback) == 1 + assert outcome.excinfo.traceback[-1].frame.code.name == 'funcfail' #def test_slavenode_dispatch_failing(self): # node = SlaveNode() From hpk at codespeak.net Tue Aug 22 15:52:11 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 15:52:11 +0200 (CEST) Subject: [py-svn] r31470 - py/branch/distributed/py/code Message-ID: <20060822135211.3D7051005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 15:52:09 2006 New Revision: 31470 Modified: py/branch/distributed/py/code/excinfo.py Log: remove unneeded reference to low level exception info. Modified: py/branch/distributed/py/code/excinfo.py ============================================================================== --- py/branch/distributed/py/code/excinfo.py (original) +++ py/branch/distributed/py/code/excinfo.py Tue Aug 22 15:52:09 2006 @@ -16,8 +16,7 @@ exprinfo = tup[1].msg if exprinfo and exprinfo.startswith('assert '): self._striptext = 'AssertionError: ' - self._excinfo = tup - self.type, self.value, tb = self._excinfo + self.type, self.value, tb = tup self.traceback = py.code.Traceback(tb) def exconly(self, tryshort=False): From hpk at codespeak.net Tue Aug 22 16:03:36 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 16:03:36 +0200 (CEST) Subject: [py-svn] r31473 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060822140336.6CB7A1005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 16:03:27 2006 New Revision: 31473 Modified: py/branch/distributed/py/test/rsession/testing/test_slavenode.py Log: (fijal, hpk) some failing tests Modified: py/branch/distributed/py/test/rsession/testing/test_slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slavenode.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slavenode.py Tue Aug 22 16:03:27 2006 @@ -30,6 +30,11 @@ assert outcome.passed assert not outcome.setupfailure + ser = outcome.serialize() + reproutcome = ReprOutcome(ser) + assert reproutcome.passed + assert not reproutcome.setupfailure + def test_slavenode_run_failing(): node = gettestnode() outcome = node.run("execitem", @@ -39,6 +44,12 @@ assert len(outcome.excinfo.traceback) == 1 assert outcome.excinfo.traceback[-1].frame.code.name == 'funcfail' + ser = outcome.serialize() + reproutcome = ReprOutcome(ser) + assert not reproutcome.passed + assert not reproutcome.setupfailure + assert reproutcome.excinfo + #def test_slavenode_dispatch_failing(self): # node = SlaveNode() # outcome = node.run("execitem", From fijal at codespeak.net Tue Aug 22 16:11:02 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 16:11:02 +0200 (CEST) Subject: [py-svn] r31474 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822141102.D57E41005A@code0.codespeak.net> Author: fijal Date: Tue Aug 22 16:10:59 2006 New Revision: 31474 Added: py/branch/distributed/py/test/rsession/outcome.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/slavenode.py py/branch/distributed/py/test/rsession/testing/test_slavenode.py Log: (hpk, fijal) - Added outcome representation for master side Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Tue Aug 22 16:10:59 2006 @@ -3,18 +3,7 @@ import py -class Outcome(object): - pass - -class FailingOutcome(Outcome): - passed = False - def __init__(self, excinfo, setupfailure=False): - self.excinfo = excinfo - self.setupfailure = setupfailure - -class PassingOutcome(Outcome): - passed = True - setupfailure = False +from py.__.test.rsession.outcome import FailingOutcome, PassingOutcome class Executor(object): def __init__(self, fun, setup=lambda: None): Added: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/outcome.py Tue Aug 22 16:10:59 2006 @@ -0,0 +1,22 @@ + +""" Classes for representing outcomes on master and slavenode sides +""" + +class Outcome(object): + def make_repr(self): + return (self.passed, self.setupfailure, repr(self.excinfo)) + +class FailingOutcome(Outcome): + passed = False + def __init__(self, excinfo, setupfailure=False): + self.excinfo = excinfo + self.setupfailure = setupfailure + +class PassingOutcome(Outcome): + passed = True + setupfailure = False + excinfo = None + +class ReprOutcome(object): + def __init__(self, repr_tuple): + self.passed, self.setupfailure, self.excinfo = repr_tuple Modified: py/branch/distributed/py/test/rsession/slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/slavenode.py (original) +++ py/branch/distributed/py/test/rsession/slavenode.py Tue Aug 22 16:10:59 2006 @@ -26,3 +26,11 @@ item = self.getitem(itemspec) ex = Executor(item.obj, setup=item.setup) return ex.execute() + + def wrapped_run(self, cmd, *args): + outcome = self.run(cmd, *args) + call = gettattr(self, "wrap_" + cmd) + return call(outcome) + + def wrap_execitem(self, outcome): + return repr(outcome) Modified: py/branch/distributed/py/test/rsession/testing/test_slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slavenode.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slavenode.py Tue Aug 22 16:10:59 2006 @@ -3,6 +3,8 @@ from py.__.test.rsession.slavenode import SlaveNode import py +from py.__.test.rsession.outcome import ReprOutcome + def funcpass(): pass @@ -30,7 +32,7 @@ assert outcome.passed assert not outcome.setupfailure - ser = outcome.serialize() + ser = outcome.make_repr() reproutcome = ReprOutcome(ser) assert reproutcome.passed assert not reproutcome.setupfailure @@ -44,7 +46,7 @@ assert len(outcome.excinfo.traceback) == 1 assert outcome.excinfo.traceback[-1].frame.code.name == 'funcfail' - ser = outcome.serialize() + ser = outcome.make_repr() reproutcome = ReprOutcome(ser) assert not reproutcome.passed assert not reproutcome.setupfailure From hpk at codespeak.net Tue Aug 22 16:13:31 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 16:13:31 +0200 (CEST) Subject: [py-svn] r31476 - py/branch/distributed/py/test/rsession Message-ID: <20060822141331.E692C10060@code0.codespeak.net> Author: hpk Date: Tue Aug 22 16:13:29 2006 New Revision: 31476 Modified: py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/outcome.py Log: (fijal, hpk) removed Outcome class hierarchy. Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Tue Aug 22 16:13:29 2006 @@ -3,7 +3,7 @@ import py -from py.__.test.rsession.outcome import FailingOutcome, PassingOutcome +from py.__.test.rsession.outcome import Outcome class Executor(object): def __init__(self, fun, setup=lambda: None): @@ -23,8 +23,8 @@ def execute(self): excinfo = self._execute(self.setup) if excinfo is not None: - return FailingOutcome(excinfo, setupfailure=True) + return Outcome(excinfo=excinfo, setupfailure=True) excinfo = self._execute(self.fun) if excinfo is not None: - return FailingOutcome(excinfo, setupfailure=False) - return PassingOutcome() + return Outcome(excinfo=excinfo, setupfailure=False) + return Outcome() Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Tue Aug 22 16:13:29 2006 @@ -3,20 +3,14 @@ """ class Outcome(object): + def __init__(self, setupfailure=False, excinfo=None): + self.passed = not excinfo + self.setupfailure = setupfailure + self.excinfo = excinfo + def make_repr(self): return (self.passed, self.setupfailure, repr(self.excinfo)) -class FailingOutcome(Outcome): - passed = False - def __init__(self, excinfo, setupfailure=False): - self.excinfo = excinfo - self.setupfailure = setupfailure - -class PassingOutcome(Outcome): - passed = True - setupfailure = False - excinfo = None - class ReprOutcome(object): def __init__(self, repr_tuple): self.passed, self.setupfailure, self.excinfo = repr_tuple From hpk at codespeak.net Tue Aug 22 16:20:22 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 16:20:22 +0200 (CEST) Subject: [py-svn] r31477 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822142022.CB2731005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 16:20:20 2006 New Revision: 31477 Modified: py/branch/distributed/py/test/rsession/slavenode.py py/branch/distributed/py/test/rsession/testing/test_slavenode.py Log: (fijal, hpk) add run method that returns serializes. Modified: py/branch/distributed/py/test/rsession/slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/slavenode.py (original) +++ py/branch/distributed/py/test/rsession/slavenode.py Tue Aug 22 16:20:20 2006 @@ -18,19 +18,11 @@ cur = next return cur - def run(self, cmd, *args): - call = getattr(self, "run_" + cmd) - return call(*args) - - def run_execitem(self, itemspec): + def execute(self, itemspec): item = self.getitem(itemspec) ex = Executor(item.obj, setup=item.setup) return ex.execute() - def wrapped_run(self, cmd, *args): - outcome = self.run(cmd, *args) - call = gettattr(self, "wrap_" + cmd) - return call(outcome) - - def wrap_execitem(self, outcome): - return repr(outcome) + def run(self, itemspec): + outcome = self.execute(itemspec) + return outcome.make_repr() Modified: py/branch/distributed/py/test/rsession/testing/test_slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slavenode.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slavenode.py Tue Aug 22 16:20:20 2006 @@ -27,8 +27,8 @@ def test_slavenode_run_passing(): node = gettestnode() - outcome = node.run("execitem", - "py/test/rsession/testing/test_slavenode.py/funcpass") + outcome = node.execute("py/test/rsession/testing/" + "test_slavenode.py/funcpass") assert outcome.passed assert not outcome.setupfailure @@ -39,8 +39,8 @@ def test_slavenode_run_failing(): node = gettestnode() - outcome = node.run("execitem", - "py/test/rsession/testing/test_slavenode.py/funcfail") + outcome = node.execute("py/test/rsession/testing/" + "test_slavenode.py/funcfail") assert not outcome.passed assert not outcome.setupfailure assert len(outcome.excinfo.traceback) == 1 @@ -52,6 +52,17 @@ assert not reproutcome.setupfailure assert reproutcome.excinfo +def test_slavenode_run_failing_wrapped(): + node = gettestnode() + repr_outcome = node.run("py/test/rsession/testing/" + "test_slavenode.py/funcfail") + outcome = ReprOutcome(repr_outcome) + assert not outcome.passed + assert not outcome.setupfailure + assert outcome.excinfo + + + #def test_slavenode_dispatch_failing(self): # node = SlaveNode() # outcome = node.run("execitem", From hpk at codespeak.net Tue Aug 22 16:45:32 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 16:45:32 +0200 (CEST) Subject: [py-svn] r31482 - py/branch/distributed/py/test/rsession Message-ID: <20060822144532.1728D1005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 16:45:28 2006 New Revision: 31482 Added: py/branch/distributed/py/test/rsession/slave_main.py (contents, props changed) Log: (fijal, hpk) draft slave_main loop Added: py/branch/distributed/py/test/rsession/slave_main.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/slave_main.py Tue Aug 22 16:45:28 2006 @@ -0,0 +1,16 @@ + +# master side: +# gw = SshGateway(sshhost) +# rsync -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s +# c = gw.remote_exec(slave_main) +# c.send(path) +# +def slave_main(receive, send): + path = receive() + node = SlaveNode(path) + while 1: + nextitem = receive() + if nextitem is None: + break + res = node.run(nextitem) + send(res) From hpk at codespeak.net Tue Aug 22 16:50:04 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 16:50:04 +0200 (CEST) Subject: [py-svn] r31483 - py/branch/distributed/py/code Message-ID: <20060822145004.59A5D1005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 16:50:00 2006 New Revision: 31483 Modified: py/branch/distributed/py/code/excinfo.py Log: reverted 31470 (it's needed apparently) Modified: py/branch/distributed/py/code/excinfo.py ============================================================================== --- py/branch/distributed/py/code/excinfo.py (original) +++ py/branch/distributed/py/code/excinfo.py Tue Aug 22 16:50:00 2006 @@ -16,7 +16,8 @@ exprinfo = tup[1].msg if exprinfo and exprinfo.startswith('assert '): self._striptext = 'AssertionError: ' - self.type, self.value, tb = tup + self._excinfo = tup + self.type, self.value, tb = self._excinfo self.traceback = py.code.Traceback(tb) def exconly(self, tryshort=False): From fijal at codespeak.net Tue Aug 22 16:55:24 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 16:55:24 +0200 (CEST) Subject: [py-svn] r31485 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822145524.3C24E1005A@code0.codespeak.net> Author: fijal Date: Tue Aug 22 16:55:11 2006 New Revision: 31485 Added: py/branch/distributed/py/test/rsession/testing/test_slavemain.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/slave_main.py py/branch/distributed/py/test/rsession/slavenode.py Log: (hpk, fijal) - added main loop for client side. Modified: py/branch/distributed/py/test/rsession/slave_main.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave_main.py (original) +++ py/branch/distributed/py/test/rsession/slave_main.py Tue Aug 22 16:55:11 2006 @@ -5,9 +5,13 @@ # c = gw.remote_exec(slave_main) # c.send(path) # +import py +from py.__.test.rsession.slavenode import SlaveNode + def slave_main(receive, send): path = receive() - node = SlaveNode(path) + rootcol = py.test.collect.Directory(path) + node = SlaveNode(rootcol) while 1: nextitem = receive() if nextitem is None: Modified: py/branch/distributed/py/test/rsession/slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/slavenode.py (original) +++ py/branch/distributed/py/test/rsession/slavenode.py Tue Aug 22 16:55:11 2006 @@ -12,7 +12,7 @@ if isinstance(namelist, str): namelist = namelist.split("/") cur = self.rootcollector - for name in namelist: + for name in namelist: next = cur.join(name) assert next is not None, (cur, name) cur = next Added: py/branch/distributed/py/test/rsession/testing/test_slavemain.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_slavemain.py Tue Aug 22 16:55:11 2006 @@ -0,0 +1,22 @@ + +""" Test main slave loop +""" + +import py +from py.__.test.rsession.slave_main import slave_main +from py.__.test.rsession.outcome import ReprOutcome + +rootdir = py.path.local(py.__file__).dirpath().dirpath() + +test_q = [None, "py/test/rsession/testing/test_slavenode.py/funcpass", + "py/test/rsession/testing/test_slavenode.py/funcfail", + str(rootdir)] + +def test_simple_test(): + res = [] + q = test_q[:] + slave_main(q.pop, res.append) + assert len(res) == 2 + res_repr = [ReprOutcome(r) for r in res] + assert not res_repr[0].passed + assert res_repr[1].passed From hpk at codespeak.net Tue Aug 22 17:36:56 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 17:36:56 +0200 (CEST) Subject: [py-svn] r31488 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822153656.89D4510050@code0.codespeak.net> Author: hpk Date: Tue Aug 22 17:36:50 2006 New Revision: 31488 Added: py/branch/distributed/py/test/rsession/masternode.py (contents, props changed) py/branch/distributed/py/test/rsession/testing/test_masternode.py (contents, props changed) Log: (fijal, hpk) bits of the masternode code. Added: py/branch/distributed/py/test/rsession/masternode.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/masternode.py Tue Aug 22 17:36:50 2006 @@ -0,0 +1,21 @@ +""" +Node code for Master. +""" +from py.__.test.rsession.outcome import ReprOutcome + +class MasterNode(object): + def __init__(self, channel, report): + self.channel = channel + self.report = report + channel.setcallback(self.receive_result) + self.pending = [] + + def receive_result(self, outcomestring): + repr_outcome = ReprOutcome(outcomestring) + item = self.pending.pop() + self.report((item, repr_outcome)) + + def send(self, item): + self.pending.insert(0, item) + itemspec = "/".join(item.listnames()) + self.channel.send(itemspec) Added: py/branch/distributed/py/test/rsession/testing/test_masternode.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_masternode.py Tue Aug 22 17:36:50 2006 @@ -0,0 +1,28 @@ +""" test master node """ + +import py +from py.__.test.rsession.masternode import MasterNode +from py.__.test.rsession.outcome import Outcome + +class DummyChannel(object): + def __init__(self): + self.sent = [] + + def setcallback(self, func): + self.callback = func + + def send(self, item): + assert py.std.marshal.dumps(item) + self.sent.append(item) + +def test_masternode(): + ch = DummyChannel() + reportlist = [] + mnode = MasterNode(ch, reportlist.append) + mnode.send(py.test.Item("ok")) + mnode.send(py.test.Item("notok")) + ch.callback(Outcome().make_repr()) + ch.callback(Outcome(excinfo=42).make_repr()) + assert len(reportlist) == 2 + assert reportlist[0][1].passed + assert not reportlist[1][1].passed From hpk at codespeak.net Tue Aug 22 17:42:45 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 17:42:45 +0200 (CEST) Subject: [py-svn] r31489 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822154245.2917910050@code0.codespeak.net> Author: hpk Date: Tue Aug 22 17:42:41 2006 New Revision: 31489 Removed: py/branch/distributed/py/test/rsession/slave_main.py py/branch/distributed/py/test/rsession/testing/test_slavemain.py Modified: py/branch/distributed/py/test/rsession/masternode.py py/branch/distributed/py/test/rsession/slavenode.py py/branch/distributed/py/test/rsession/testing/test_slavenode.py Log: (fijal, hpk) removing one file. Modified: py/branch/distributed/py/test/rsession/masternode.py ============================================================================== --- py/branch/distributed/py/test/rsession/masternode.py (original) +++ py/branch/distributed/py/test/rsession/masternode.py Tue Aug 22 17:42:41 2006 @@ -19,3 +19,12 @@ self.pending.insert(0, item) itemspec = "/".join(item.listnames()) self.channel.send(itemspec) + +# master side: +# gw = SshGateway(sshhost) +# rsync -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s +# c = gw.remote_exec(slave_main) +# c.send(path) +# + + Deleted: /py/branch/distributed/py/test/rsession/slave_main.py ============================================================================== --- /py/branch/distributed/py/test/rsession/slave_main.py Tue Aug 22 17:42:41 2006 +++ (empty file) @@ -1,20 +0,0 @@ - -# master side: -# gw = SshGateway(sshhost) -# rsync -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s -# c = gw.remote_exec(slave_main) -# c.send(path) -# -import py -from py.__.test.rsession.slavenode import SlaveNode - -def slave_main(receive, send): - path = receive() - rootcol = py.test.collect.Directory(path) - node = SlaveNode(rootcol) - while 1: - nextitem = receive() - if nextitem is None: - break - res = node.run(nextitem) - send(res) Modified: py/branch/distributed/py/test/rsession/slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/slavenode.py (original) +++ py/branch/distributed/py/test/rsession/slavenode.py Tue Aug 22 17:42:41 2006 @@ -2,6 +2,7 @@ Node code for slaves. """ +import py from py.__.test.rsession.executor import Executor class SlaveNode(object): @@ -26,3 +27,14 @@ def run(self, itemspec): outcome = self.execute(itemspec) return outcome.make_repr() + +def slave_main(receive, send): + path = receive() + rootcol = py.test.collect.Directory(path) + node = SlaveNode(rootcol) + while 1: + nextitem = receive() + if nextitem is None: + break + res = node.run(nextitem) + send(res) Deleted: /py/branch/distributed/py/test/rsession/testing/test_slavemain.py ============================================================================== --- /py/branch/distributed/py/test/rsession/testing/test_slavemain.py Tue Aug 22 17:42:41 2006 +++ (empty file) @@ -1,22 +0,0 @@ - -""" Test main slave loop -""" - -import py -from py.__.test.rsession.slave_main import slave_main -from py.__.test.rsession.outcome import ReprOutcome - -rootdir = py.path.local(py.__file__).dirpath().dirpath() - -test_q = [None, "py/test/rsession/testing/test_slavenode.py/funcpass", - "py/test/rsession/testing/test_slavenode.py/funcfail", - str(rootdir)] - -def test_simple_test(): - res = [] - q = test_q[:] - slave_main(q.pop, res.append) - assert len(res) == 2 - res_repr = [ReprOutcome(r) for r in res] - assert not res_repr[0].passed - assert res_repr[1].passed Modified: py/branch/distributed/py/test/rsession/testing/test_slavenode.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slavenode.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slavenode.py Tue Aug 22 17:42:41 2006 @@ -1,9 +1,11 @@ """ Testing the slave side node code (in a local way). """ -from py.__.test.rsession.slavenode import SlaveNode +from py.__.test.rsession.slavenode import SlaveNode, slave_main +from py.__.test.rsession.outcome import ReprOutcome import py -from py.__.test.rsession.outcome import ReprOutcome +def setup_module(mod): + mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() def funcpass(): pass @@ -12,7 +14,6 @@ raise AssertionError("hello world") def gettestnode(): - rootdir = py.path.local(py.__file__).dirpath().dirpath() rootcol = py.test.collect.Directory(rootdir) node = SlaveNode(rootcol) return node @@ -61,13 +62,16 @@ assert not outcome.setupfailure assert outcome.excinfo +def test_slave_main_simple(): + res = [] + q = [None, + "py/test/rsession/testing/test_slavenode.py/funcpass", + "py/test/rsession/testing/test_slavenode.py/funcfail", + str(rootdir)] + slave_main(q.pop, res.append) + assert len(res) == 2 + res_repr = [ReprOutcome(r) for r in res] + assert not res_repr[0].passed + assert res_repr[1].passed -#def test_slavenode_dispatch_failing(self): -# node = SlaveNode() -# outcome = node.run("execitem", -# "py.__.test.rsession.testing.test_slavenode.funcpass" -# .split('.'), -# ) -# assert not outcome.haspassed() -# assert outcome.excinfo.type == AssertionError From hpk at codespeak.net Tue Aug 22 17:44:35 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 17:44:35 +0200 (CEST) Subject: [py-svn] r31490 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822154435.9AA2010050@code0.codespeak.net> Author: hpk Date: Tue Aug 22 17:44:29 2006 New Revision: 31490 Added: py/branch/distributed/py/test/rsession/master.py - copied unchanged from r31489, py/branch/distributed/py/test/rsession/masternode.py py/branch/distributed/py/test/rsession/slave.py - copied unchanged from r31489, py/branch/distributed/py/test/rsession/slavenode.py py/branch/distributed/py/test/rsession/testing/test_master.py - copied, changed from r31488, py/branch/distributed/py/test/rsession/testing/test_masternode.py py/branch/distributed/py/test/rsession/testing/test_slave.py - copied, changed from r31489, py/branch/distributed/py/test/rsession/testing/test_slavenode.py Removed: py/branch/distributed/py/test/rsession/masternode.py py/branch/distributed/py/test/rsession/slavenode.py py/branch/distributed/py/test/rsession/testing/test_masternode.py py/branch/distributed/py/test/rsession/testing/test_slavenode.py Log: (fijal, hpk) refactor towards nice slave and master naming. Deleted: /py/branch/distributed/py/test/rsession/masternode.py ============================================================================== --- /py/branch/distributed/py/test/rsession/masternode.py Tue Aug 22 17:44:29 2006 +++ (empty file) @@ -1,30 +0,0 @@ -""" -Node code for Master. -""" -from py.__.test.rsession.outcome import ReprOutcome - -class MasterNode(object): - def __init__(self, channel, report): - self.channel = channel - self.report = report - channel.setcallback(self.receive_result) - self.pending = [] - - def receive_result(self, outcomestring): - repr_outcome = ReprOutcome(outcomestring) - item = self.pending.pop() - self.report((item, repr_outcome)) - - def send(self, item): - self.pending.insert(0, item) - itemspec = "/".join(item.listnames()) - self.channel.send(itemspec) - -# master side: -# gw = SshGateway(sshhost) -# rsync -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s -# c = gw.remote_exec(slave_main) -# c.send(path) -# - - Deleted: /py/branch/distributed/py/test/rsession/slavenode.py ============================================================================== --- /py/branch/distributed/py/test/rsession/slavenode.py Tue Aug 22 17:44:29 2006 +++ (empty file) @@ -1,40 +0,0 @@ -""" -Node code for slaves. -""" - -import py -from py.__.test.rsession.executor import Executor - -class SlaveNode(object): - def __init__(self, rootcollector): - self.rootcollector = rootcollector - - def getitem(self, namelist): - if isinstance(namelist, str): - namelist = namelist.split("/") - cur = self.rootcollector - for name in namelist: - next = cur.join(name) - assert next is not None, (cur, name) - cur = next - return cur - - def execute(self, itemspec): - item = self.getitem(itemspec) - ex = Executor(item.obj, setup=item.setup) - return ex.execute() - - def run(self, itemspec): - outcome = self.execute(itemspec) - return outcome.make_repr() - -def slave_main(receive, send): - path = receive() - rootcol = py.test.collect.Directory(path) - node = SlaveNode(rootcol) - while 1: - nextitem = receive() - if nextitem is None: - break - res = node.run(nextitem) - send(res) Copied: py/branch/distributed/py/test/rsession/testing/test_master.py (from r31488, py/branch/distributed/py/test/rsession/testing/test_masternode.py) ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_masternode.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Tue Aug 22 17:44:29 2006 @@ -1,7 +1,7 @@ """ test master node """ import py -from py.__.test.rsession.masternode import MasterNode +from py.__.test.rsession.master import MasterNode from py.__.test.rsession.outcome import Outcome class DummyChannel(object): Deleted: /py/branch/distributed/py/test/rsession/testing/test_masternode.py ============================================================================== --- /py/branch/distributed/py/test/rsession/testing/test_masternode.py Tue Aug 22 17:44:29 2006 +++ (empty file) @@ -1,28 +0,0 @@ -""" test master node """ - -import py -from py.__.test.rsession.masternode import MasterNode -from py.__.test.rsession.outcome import Outcome - -class DummyChannel(object): - def __init__(self): - self.sent = [] - - def setcallback(self, func): - self.callback = func - - def send(self, item): - assert py.std.marshal.dumps(item) - self.sent.append(item) - -def test_masternode(): - ch = DummyChannel() - reportlist = [] - mnode = MasterNode(ch, reportlist.append) - mnode.send(py.test.Item("ok")) - mnode.send(py.test.Item("notok")) - ch.callback(Outcome().make_repr()) - ch.callback(Outcome(excinfo=42).make_repr()) - assert len(reportlist) == 2 - assert reportlist[0][1].passed - assert not reportlist[1][1].passed Copied: py/branch/distributed/py/test/rsession/testing/test_slave.py (from r31489, py/branch/distributed/py/test/rsession/testing/test_slavenode.py) ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slavenode.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Tue Aug 22 17:44:29 2006 @@ -1,6 +1,6 @@ """ Testing the slave side node code (in a local way). """ -from py.__.test.rsession.slavenode import SlaveNode, slave_main +from py.__.test.rsession.slave import SlaveNode, slave_main from py.__.test.rsession.outcome import ReprOutcome import py @@ -18,18 +18,18 @@ node = SlaveNode(rootcol) return node -def test_slavenode_getitem(): +def test_slave_getitem(): node = gettestnode() - item = node.getitem("py/test/rsession/testing/test_slavenode.py/funcpass") + item = node.getitem("py/test/rsession/testing/test_slave.py/funcpass") assert isinstance(item, py.test.Item) assert item.name == 'funcpass' res = item.run() assert res is None -def test_slavenode_run_passing(): +def test_slave_run_passing(): node = gettestnode() outcome = node.execute("py/test/rsession/testing/" - "test_slavenode.py/funcpass") + "test_slave.py/funcpass") assert outcome.passed assert not outcome.setupfailure @@ -38,10 +38,10 @@ assert reproutcome.passed assert not reproutcome.setupfailure -def test_slavenode_run_failing(): +def test_slave_run_failing(): node = gettestnode() outcome = node.execute("py/test/rsession/testing/" - "test_slavenode.py/funcfail") + "test_slave.py/funcfail") assert not outcome.passed assert not outcome.setupfailure assert len(outcome.excinfo.traceback) == 1 @@ -53,10 +53,10 @@ assert not reproutcome.setupfailure assert reproutcome.excinfo -def test_slavenode_run_failing_wrapped(): +def test_slave_run_failing_wrapped(): node = gettestnode() repr_outcome = node.run("py/test/rsession/testing/" - "test_slavenode.py/funcfail") + "test_slave.py/funcfail") outcome = ReprOutcome(repr_outcome) assert not outcome.passed assert not outcome.setupfailure @@ -65,8 +65,8 @@ def test_slave_main_simple(): res = [] q = [None, - "py/test/rsession/testing/test_slavenode.py/funcpass", - "py/test/rsession/testing/test_slavenode.py/funcfail", + "py/test/rsession/testing/test_slave.py/funcpass", + "py/test/rsession/testing/test_slave.py/funcfail", str(rootdir)] slave_main(q.pop, res.append) assert len(res) == 2 Deleted: /py/branch/distributed/py/test/rsession/testing/test_slavenode.py ============================================================================== --- /py/branch/distributed/py/test/rsession/testing/test_slavenode.py Tue Aug 22 17:44:29 2006 +++ (empty file) @@ -1,77 +0,0 @@ - -""" Testing the slave side node code (in a local way). """ -from py.__.test.rsession.slavenode import SlaveNode, slave_main -from py.__.test.rsession.outcome import ReprOutcome -import py - -def setup_module(mod): - mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() - -def funcpass(): - pass - -def funcfail(): - raise AssertionError("hello world") - -def gettestnode(): - rootcol = py.test.collect.Directory(rootdir) - node = SlaveNode(rootcol) - return node - -def test_slavenode_getitem(): - node = gettestnode() - item = node.getitem("py/test/rsession/testing/test_slavenode.py/funcpass") - assert isinstance(item, py.test.Item) - assert item.name == 'funcpass' - res = item.run() - assert res is None - -def test_slavenode_run_passing(): - node = gettestnode() - outcome = node.execute("py/test/rsession/testing/" - "test_slavenode.py/funcpass") - assert outcome.passed - assert not outcome.setupfailure - - ser = outcome.make_repr() - reproutcome = ReprOutcome(ser) - assert reproutcome.passed - assert not reproutcome.setupfailure - -def test_slavenode_run_failing(): - node = gettestnode() - outcome = node.execute("py/test/rsession/testing/" - "test_slavenode.py/funcfail") - assert not outcome.passed - assert not outcome.setupfailure - assert len(outcome.excinfo.traceback) == 1 - assert outcome.excinfo.traceback[-1].frame.code.name == 'funcfail' - - ser = outcome.make_repr() - reproutcome = ReprOutcome(ser) - assert not reproutcome.passed - assert not reproutcome.setupfailure - assert reproutcome.excinfo - -def test_slavenode_run_failing_wrapped(): - node = gettestnode() - repr_outcome = node.run("py/test/rsession/testing/" - "test_slavenode.py/funcfail") - outcome = ReprOutcome(repr_outcome) - assert not outcome.passed - assert not outcome.setupfailure - assert outcome.excinfo - -def test_slave_main_simple(): - res = [] - q = [None, - "py/test/rsession/testing/test_slavenode.py/funcpass", - "py/test/rsession/testing/test_slavenode.py/funcfail", - str(rootdir)] - slave_main(q.pop, res.append) - assert len(res) == 2 - res_repr = [ReprOutcome(r) for r in res] - assert not res_repr[0].passed - assert res_repr[1].passed - - From fijal at codespeak.net Tue Aug 22 18:18:33 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 18:18:33 +0200 (CEST) Subject: [py-svn] r31491 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822161833.616DD1005A@code0.codespeak.net> Author: fijal Date: Tue Aug 22 18:18:30 2006 New Revision: 31491 Added: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/master.py Log: (hpk, fijal) - Added dispatching loop. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Tue Aug 22 18:18:30 2006 @@ -3,6 +3,8 @@ """ from py.__.test.rsession.outcome import ReprOutcome +MAX_TASKS_PER_NODE = 5 + class MasterNode(object): def __init__(self, channel, report): self.channel = channel @@ -20,6 +22,26 @@ itemspec = "/".join(item.listnames()) self.channel.send(itemspec) +class RSession(object): + def __init__(self): + pass + +def dispatch_loop(masternodes, itemgenerator, shouldstop, event, + maxtasks_per_node = MAX_TASKS_PER_NODE): + while 1: + try: + for node in masternodes: + if len(node.pending) < maxtasks_per_node: + item = itemgenerator.next() + if shouldstop(): + break + node.send(item) + except StopIteration: + break + event.wait() + event.clear() + # wait here + # master side: # gw = SshGateway(sshhost) # rsync -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s Added: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Tue Aug 22 18:18:30 2006 @@ -0,0 +1,36 @@ + +""" test dispatching stuff for making 1-n master -> slave connection +and test it locally +""" + +import time, threading + +from py.__.test.rsession.master import dispatch_loop + +#def sleeping_test(): +# time.sleep(0.3) + +class DummyMasterNode(object): + def __init__(self): + self.pending = [] + + def send(self, data): + self.pending.append(data) + +class DummyEvent(object): + def __init__(self, nodes): + self.nodes = nodes + + def wait(self): + for node in self.nodes: + node.pending.pop() + + def clear(self): + pass + +def test_dispatch_loop(): + masternodes = [DummyMasterNode(), DummyMasterNode()] + event = DummyEvent(masternodes) + itemgenerator = iter(range(100)) + shouldstop = lambda : False + dispatch_loop(masternodes, itemgenerator, shouldstop, event) From hpk at codespeak.net Tue Aug 22 18:48:45 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 18:48:45 +0200 (CEST) Subject: [py-svn] r31497 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060822164845.8C53410069@code0.codespeak.net> Author: hpk Date: Tue Aug 22 18:48:43 2006 New Revision: 31497 Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py Log: factor out the item specs. Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Tue Aug 22 18:48:43 2006 @@ -7,12 +7,19 @@ def setup_module(mod): mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() +# ---------------------------------------------------------------------- +# inlined testing functions used below def funcpass(): pass def funcfail(): raise AssertionError("hello world") +funcpass_spec = "py/test/rsession/testing/test_slave.py/funcpass" +funcfail_spec = "py/test/rsession/testing/test_slave.py/funcfail" +# ---------------------------------------------------------------------- + + def gettestnode(): rootcol = py.test.collect.Directory(rootdir) node = SlaveNode(rootcol) @@ -20,7 +27,7 @@ def test_slave_getitem(): node = gettestnode() - item = node.getitem("py/test/rsession/testing/test_slave.py/funcpass") + item = node.getitem(funcpass_spec) assert isinstance(item, py.test.Item) assert item.name == 'funcpass' res = item.run() @@ -28,8 +35,7 @@ def test_slave_run_passing(): node = gettestnode() - outcome = node.execute("py/test/rsession/testing/" - "test_slave.py/funcpass") + outcome = node.execute(funcpass_spec) assert outcome.passed assert not outcome.setupfailure @@ -40,8 +46,7 @@ def test_slave_run_failing(): node = gettestnode() - outcome = node.execute("py/test/rsession/testing/" - "test_slave.py/funcfail") + outcome = node.execute(funcfail_spec) assert not outcome.passed assert not outcome.setupfailure assert len(outcome.excinfo.traceback) == 1 @@ -55,8 +60,7 @@ def test_slave_run_failing_wrapped(): node = gettestnode() - repr_outcome = node.run("py/test/rsession/testing/" - "test_slave.py/funcfail") + repr_outcome = node.run(funcfail_spec) outcome = ReprOutcome(repr_outcome) assert not outcome.passed assert not outcome.setupfailure @@ -65,8 +69,8 @@ def test_slave_main_simple(): res = [] q = [None, - "py/test/rsession/testing/test_slave.py/funcpass", - "py/test/rsession/testing/test_slave.py/funcfail", + funcpass_spec, + funcfail_spec, str(rootdir)] slave_main(q.pop, res.append) assert len(res) == 2 From fijal at codespeak.net Tue Aug 22 18:53:17 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 18:53:17 +0200 (CEST) Subject: [py-svn] r31498 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822165317.CA9381006F@code0.codespeak.net> Author: fijal Date: Tue Aug 22 18:53:15 2006 New Revision: 31498 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Log: (hpk, fijal) - Added slave setup code. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Tue Aug 22 18:53:15 2006 @@ -1,6 +1,7 @@ """ Node code for Master. """ +import py from py.__.test.rsession.outcome import ReprOutcome MAX_TASKS_PER_NODE = 5 @@ -42,6 +43,17 @@ event.clear() # wait here +def setup(): + from py.__.test.rsession.slave import slave_main + slave_main(channel.receive, channel.send) + +def setup_slave(gateway, base_path = None): + if base_path is None: + base_path = str(py.path.local(py.__file__).dirpath().dirpath()) + ch = gateway.remote_exec(py.code.Source(setup, "setup()")) + ch.send(base_path) + return ch + # master side: # gw = SshGateway(sshhost) # rsync -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s Modified: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_dispatcher.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Tue Aug 22 18:53:15 2006 @@ -5,10 +5,13 @@ import time, threading -from py.__.test.rsession.master import dispatch_loop +import py +from py.__.test.rsession.master import dispatch_loop, setup_slave +from py.__.test.rsession.outcome import ReprOutcome +from py.__.test.rsession.testing.test_slave import funcpass_spec -#def sleeping_test(): -# time.sleep(0.3) +def setup_module(mod): + mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() class DummyMasterNode(object): def __init__(self): @@ -34,3 +37,18 @@ itemgenerator = iter(range(100)) shouldstop = lambda : False dispatch_loop(masternodes, itemgenerator, shouldstop, event) + +def test_slave_setup(): + gw = py.execnet.PopenGateway() + channel = setup_slave(gw) + channel.send(funcpass_spec) + output = ReprOutcome(channel.receive()) + assert output.passed + channel.send(None) + channel.waitclose(10) + gw.exit() + +#def test_dispatcher_real(): +# event = threading.Event() +# gw1 = py.execnet.PopenGateway() +# gw2 = py.execnet.PopenGateway() From hpk at codespeak.net Tue Aug 22 19:01:58 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 19:01:58 +0200 (CEST) Subject: [py-svn] r31499 - py/branch/distributed/py/test/rsession Message-ID: <20060822170158.5BF8110069@code0.codespeak.net> Author: hpk Date: Tue Aug 22 19:01:55 2006 New Revision: 31499 Modified: py/branch/distributed/py/test/rsession/master.py Log: cleanup a bit. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Tue Aug 22 19:01:55 2006 @@ -23,12 +23,8 @@ itemspec = "/".join(item.listnames()) self.channel.send(itemspec) -class RSession(object): - def __init__(self): - pass - def dispatch_loop(masternodes, itemgenerator, shouldstop, event, - maxtasks_per_node = MAX_TASKS_PER_NODE): + maxtasks_per_node = MAX_TASKS_PER_NODE): while 1: try: for node in masternodes: @@ -41,13 +37,12 @@ break event.wait() event.clear() - # wait here def setup(): from py.__.test.rsession.slave import slave_main slave_main(channel.receive, channel.send) -def setup_slave(gateway, base_path = None): +def setup_slave(gateway, base_path=None): if base_path is None: base_path = str(py.path.local(py.__file__).dirpath().dirpath()) ch = gateway.remote_exec(py.code.Source(setup, "setup()")) From hpk at codespeak.net Tue Aug 22 19:48:51 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 19:48:51 +0200 (CEST) Subject: [py-svn] r31502 - in py/branch/distributed/py/test: . rsession rsession/testing testing Message-ID: <20060822174851.8934B10050@code0.codespeak.net> Author: hpk Date: Tue Aug 22 19:48:47 2006 New Revision: 31502 Modified: py/branch/distributed/py/test/collect.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_slave.py py/branch/distributed/py/test/testing/test_collect.py Log: Introduce a getitembynames method for collectors. Modified: py/branch/distributed/py/test/collect.py ============================================================================== --- py/branch/distributed/py/test/collect.py (original) +++ py/branch/distributed/py/test/collect.py Tue Aug 22 19:48:47 2006 @@ -147,6 +147,16 @@ def listnames(self): return [x.name for x in self.listchain()] + def getitembynames(self, namelist): + if isinstance(namelist, str): + namelist = namelist.split("/") + cur = self + for name in namelist: + next = cur.join(name) + assert next is not None, (cur, name) + cur = next + return cur + def haskeyword(self, keyword): return keyword in self.name Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Tue Aug 22 19:48:47 2006 @@ -9,18 +9,8 @@ def __init__(self, rootcollector): self.rootcollector = rootcollector - def getitem(self, namelist): - if isinstance(namelist, str): - namelist = namelist.split("/") - cur = self.rootcollector - for name in namelist: - next = cur.join(name) - assert next is not None, (cur, name) - cur = next - return cur - def execute(self, itemspec): - item = self.getitem(itemspec) + item = self.rootcollector.getitembynames(itemspec) ex = Executor(item.obj, setup=item.setup) return ex.execute() Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Tue Aug 22 19:48:47 2006 @@ -25,14 +25,6 @@ node = SlaveNode(rootcol) return node -def test_slave_getitem(): - node = gettestnode() - item = node.getitem(funcpass_spec) - assert isinstance(item, py.test.Item) - assert item.name == 'funcpass' - res = item.run() - assert res is None - def test_slave_run_passing(): node = gettestnode() outcome = node.execute(funcpass_spec) Modified: py/branch/distributed/py/test/testing/test_collect.py ============================================================================== --- py/branch/distributed/py/test/testing/test_collect.py (original) +++ py/branch/distributed/py/test/testing/test_collect.py Tue Aug 22 19:48:47 2006 @@ -24,6 +24,17 @@ assert cur.parent == col.parent assert cur.fspath == cur.fspath +def test_collect_listnames_and_back(): + col1 = py.test.collect.Directory(datadir.dirpath()) + col2 = col1.join(datadir.basename) + col3 = col2.join('filetest.py') + l = col3.listnames() + assert len(l) == 3 + x = col1.getitembynames(l[1:]) + assert x.name == "filetest.py" + x = col1.getitembynames("/".join(l[1:])) + assert x.name == "filetest.py" + def test_finds_tests(): fn = datadir / 'filetest.py' col = py.test.collect.Module(fn) From fijal at codespeak.net Tue Aug 22 20:07:40 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 20:07:40 +0200 (CEST) Subject: [py-svn] r31503 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822180740.DE7DA1005A@code0.codespeak.net> Author: fijal Date: Tue Aug 22 20:07:38 2006 New Revision: 31503 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Log: (hpk, fijal) - Added test runnin popen. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Tue Aug 22 20:07:38 2006 @@ -20,7 +20,7 @@ def send(self, item): self.pending.insert(0, item) - itemspec = "/".join(item.listnames()) + itemspec = "/".join(item.listnames()[1:]) self.channel.send(itemspec) def dispatch_loop(masternodes, itemgenerator, shouldstop, event, @@ -35,7 +35,7 @@ node.send(item) except StopIteration: break - event.wait() + event.wait(3) event.clear() def setup(): @@ -43,6 +43,7 @@ slave_main(channel.receive, channel.send) def setup_slave(gateway, base_path=None): + # XXX: Sending path from master side, should be guessed on remote side if base_path is None: base_path = str(py.path.local(py.__file__).dirpath().dirpath()) ch = gateway.remote_exec(py.code.Source(setup, "setup()")) @@ -55,5 +56,3 @@ # c = gw.remote_exec(slave_main) # c.send(path) # - - Modified: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_dispatcher.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Tue Aug 22 20:07:38 2006 @@ -6,9 +6,9 @@ import time, threading import py -from py.__.test.rsession.master import dispatch_loop, setup_slave +from py.__.test.rsession.master import dispatch_loop, setup_slave, MasterNode from py.__.test.rsession.outcome import ReprOutcome -from py.__.test.rsession.testing.test_slave import funcpass_spec +from py.__.test.rsession.testing.test_slave import funcpass_spec, funcfail_spec def setup_module(mod): mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() @@ -24,7 +24,7 @@ def __init__(self, nodes): self.nodes = nodes - def wait(self): + def wait(self, timeout = None): for node in self.nodes: node.pending.pop() @@ -48,7 +48,28 @@ channel.waitclose(10) gw.exit() -#def test_dispatcher_real(): -# event = threading.Event() -# gw1 = py.execnet.PopenGateway() -# gw2 = py.execnet.PopenGateway() +def test_slave_running(): + event = threading.Event() + + def simple_reporter(result): + event.set() + item, res = result + if item.code.name == 'funcpass': + assert res.passed + else: + assert not res.passed + + def open_gw(): + gw = py.execnet.PopenGateway() + channel = setup_slave(gw) + mn = MasterNode(channel, simple_reporter) + return mn + + master_nodes = [open_gw(), open_gw(), open_gw()] + rootcol = py.test.collect.Directory(rootdir) + funcpass_item = rootcol.getitembynames(funcpass_spec) + funcfail_item = rootcol.getitembynames(funcfail_spec) + itemgenerator = iter([funcfail_item] + [funcpass_item] * 5 + [funcfail_item] * 5) + shouldstop = lambda : False + dispatch_loop(master_nodes, itemgenerator, shouldstop, event) + From fijal at codespeak.net Tue Aug 22 20:17:19 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 20:17:19 +0200 (CEST) Subject: [py-svn] r31504 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822181719.D20261005A@code0.codespeak.net> Author: fijal Date: Tue Aug 22 20:17:17 2006 New Revision: 31504 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_dispatcher.py py/branch/distributed/py/test/rsession/testing/test_slave.py Log: (hpk, fijal) - Refactored a little Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Tue Aug 22 20:17:17 2006 @@ -38,16 +38,10 @@ event.wait(3) event.clear() -def setup(): - from py.__.test.rsession.slave import slave_main - slave_main(channel.receive, channel.send) - -def setup_slave(gateway, base_path=None): - # XXX: Sending path from master side, should be guessed on remote side - if base_path is None: - base_path = str(py.path.local(py.__file__).dirpath().dirpath()) +def setup_slave(gateway): + from py.__.test.rsession.slave import setup + ch = gateway.remote_exec(py.code.Source(setup, "setup()")) - ch.send(base_path) return ch # master side: Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Tue Aug 22 20:17:17 2006 @@ -18,8 +18,7 @@ outcome = self.execute(itemspec) return outcome.make_repr() -def slave_main(receive, send): - path = receive() +def slave_main(receive, send, path): rootcol = py.test.collect.Directory(path) node = SlaveNode(rootcol) while 1: @@ -28,3 +27,11 @@ break res = node.run(nextitem) send(res) + +def setup(): + # XXX: let's assume that we're always testing relatively to pylib dir + import py + from py.__.test.rsession.slave import slave_main + + base_path = str(py.path.local(py.__file__).dirpath().dirpath()) + slave_main(channel.receive, channel.send, base_path) Modified: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_dispatcher.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Tue Aug 22 20:17:17 2006 @@ -72,4 +72,3 @@ itemgenerator = iter([funcfail_item] + [funcpass_item] * 5 + [funcfail_item] * 5) shouldstop = lambda : False dispatch_loop(master_nodes, itemgenerator, shouldstop, event) - Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Tue Aug 22 20:17:17 2006 @@ -62,9 +62,9 @@ res = [] q = [None, funcpass_spec, - funcfail_spec, - str(rootdir)] - slave_main(q.pop, res.append) + funcfail_spec + ] + slave_main(q.pop, res.append, str(rootdir)) assert len(res) == 2 res_repr = [ReprOutcome(r) for r in res] assert not res_repr[0].passed From hpk at codespeak.net Tue Aug 22 20:37:26 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 20:37:26 +0200 (CEST) Subject: [py-svn] r31505 - in py/branch/distributed/py/execnet: . testing Message-ID: <20060822183726.D291E1005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 20:37:22 2006 New Revision: 31505 Modified: py/branch/distributed/py/execnet/register.py py/branch/distributed/py/execnet/testing/test_gateway.py Log: add sshaddress as an attribute to SshGateway Modified: py/branch/distributed/py/execnet/register.py ============================================================================== --- py/branch/distributed/py/execnet/register.py (original) +++ py/branch/distributed/py/execnet/register.py Tue Aug 22 20:37:22 2006 @@ -139,6 +139,7 @@ class SshGateway(PopenCmdGateway): def __init__(self, sshaddress, remotepython='python', identity=None): + self.sshaddress = sshaddress remotecmd = '%s -u -c "exec input()"' % (remotepython,) cmdline = [sshaddress, remotecmd] # XXX Unix style quoting Modified: py/branch/distributed/py/execnet/testing/test_gateway.py ============================================================================== --- py/branch/distributed/py/execnet/testing/test_gateway.py (original) +++ py/branch/distributed/py/execnet/testing/test_gateway.py Tue Aug 22 20:37:22 2006 @@ -388,3 +388,6 @@ py.test.skip("no known ssh target, use -S to set one") cls.gw = py.execnet.SshGateway(option.sshtarget) + def test_sshaddress(self): + assert self.gw.sshaddress == option.sshtarget + From hpk at codespeak.net Tue Aug 22 20:49:57 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 20:49:57 +0200 (CEST) Subject: [py-svn] r31506 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060822184957.DD45F1005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 20:49:54 2006 New Revision: 31506 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Log: (fijal, hpk) reintroduce sending of (a relative) path for setting up slaves. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Tue Aug 22 20:49:54 2006 @@ -35,18 +35,34 @@ node.send(item) except StopIteration: break - event.wait(3) + event.wait(3) # XXX argh event.clear() -def setup_slave(gateway): - from py.__.test.rsession.slave import setup - - ch = gateway.remote_exec(py.code.Source(setup, "setup()")) - return ch +_rsync = py.path.local.sysfind("rsync") +def bin_rsync(source, sshaddress, destpath): + rsync.execute("-az", "--delete", "--exclude=.svn", + source, sshaddress + ":" + str(destpath)) + +def setup_slave(gateway, destpath): + from py.__.test.rsession import slave + ch = gateway.remote_exec(py.code.Source(slave.setup, "setup()")) + ch.send(str(destpath)) + return ch + +#def setup_(gateway, rootdir, report, path="pytest_cache"): +# bin_rsync(rootdir, gateway.sshaddress, path) +# return MasterNode(ch, report) +# +#def main(): + # master side: # gw = SshGateway(sshhost) -# rsync -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s +# +# setup_slave(gw) +# path = "pytest_cache" +# rsync(rootdir, gw.sshhost, path) +# pytest_cache -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s # c = gw.remote_exec(slave_main) # c.send(path) # Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Tue Aug 22 20:49:54 2006 @@ -29,9 +29,9 @@ send(res) def setup(): - # XXX: let's assume that we're always testing relatively to pylib dir - import py + base_path = channel.receive() # path is ready + import os, sys + base_path = os.path.abspath(base_path) + sys.path.insert(0, base_path) from py.__.test.rsession.slave import slave_main - - base_path = str(py.path.local(py.__file__).dirpath().dirpath()) slave_main(channel.receive, channel.send, base_path) Modified: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_dispatcher.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Tue Aug 22 20:49:54 2006 @@ -40,7 +40,7 @@ def test_slave_setup(): gw = py.execnet.PopenGateway() - channel = setup_slave(gw) + channel = setup_slave(gw, rootdir) channel.send(funcpass_spec) output = ReprOutcome(channel.receive()) assert output.passed @@ -61,7 +61,7 @@ def open_gw(): gw = py.execnet.PopenGateway() - channel = setup_slave(gw) + channel = setup_slave(gw, rootdir) mn = MasterNode(channel, simple_reporter) return mn From hpk at codespeak.net Tue Aug 22 21:09:47 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 21:09:47 +0200 (CEST) Subject: [py-svn] r31507 - in py/branch/distributed/py/test: rsession testing Message-ID: <20060822190947.AF26A1005A@code0.codespeak.net> Author: hpk Date: Tue Aug 22 21:09:39 2006 New Revision: 31507 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/testing/test_collect.py Log: (fijal, hpk) hacking and cleaning up a bit to make RSession work a bit. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Tue Aug 22 21:09:39 2006 @@ -41,8 +41,9 @@ _rsync = py.path.local.sysfind("rsync") def bin_rsync(source, sshaddress, destpath): - rsync.execute("-az", "--delete", "--exclude=.svn", - source, sshaddress + ":" + str(destpath)) + print "rsyncing", source, "to", sshaddress, destpath + _rsync.sysexec("-az", "--delete", "--exclude=.svn", + str(source), sshaddress + ":" + str(destpath)) def setup_slave(gateway, destpath): from py.__.test.rsession import slave Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Tue Aug 22 21:09:39 2006 @@ -4,14 +4,6 @@ def __init__(self, config): self.config = config - def start(self, colitem): - print "starting colitem", colitem - - def finish(self, colitem, outcome): - print "finishing colitem", colitem, "outcome", outcome - - - class RSession(object): """ An RSession executes collectors/items through a runner. @@ -19,110 +11,24 @@ """ def __init__(self, config): self.config = config - self.report = Reporter(config) + + def report(self, *whatever): + print whatever def main(self, args): """ main loop for running tests. """ - colitems = self._map2colitems(args) - print "hello world" - print colitems - raise SystemExit, 1 - count = 0 - while colitems: - colitem = colitems.pop() - self.runtraced(colitem) - if len(self.results) < count: - time.sleep(0.1) - - def runtraced(self, colitem): - outcome = None - colitem.startcapture() - try: - self.report.predispatch(colitem) - self.dispatch(colitem) - self.report.postdispatch(colitem) - try: - try: - outcome = self.run(colitem) - except (KeyboardInterrupt, Exit): - raise - except colitem.Outcome, outcome: - if outcome.excinfo is None: - outcome.excinfo = py.code.ExceptionInfo() - except: - excinfo = py.code.ExceptionInfo() - outcome = colitem.Failed(excinfo=excinfo) - assert (outcome is None or - isinstance(outcome, (list, colitem.Outcome))) - finally: - self.finish(colitem, outcome) - if isinstance(outcome, colitem.Failed) and self.config.option.exitfirst: - py.test.exit("exit on first problem configured.", item=colitem) - finally: - colitem.finishcapture() - - def run(self, colitem): - if self.config.option.collectonly and isinstance(colitem, py.test.Item): - return - if isinstance(colitem, py.test.Item): - self.skipbykeyword(colitem) - res = colitem.run() - if res is None: - return py.test.Item.Passed() - elif not isinstance(res, (list, tuple)): - raise TypeError("%r.run() returned neither " - "list, tuple nor None: %r" % (colitem, res)) - else: - finish = self.startiteration(colitem, res) - try: - for name in res: - obj = colitem.join(name) - assert obj is not None - self.runtraced(obj) - finally: - if finish: - finish() - return res - - def skipbykeyword(self, colitem): - keyword = self.config.option.keyword - if not keyword: - return - chain = colitem.listchain() - for key in filter(None, keyword.split()): - eor = key[:1] == '-' - if eor: - key = key[1:] - if not (eor ^ self._matchonekeyword(key, chain)): - py.test.skip("test not selected by keyword %r" %(keyword,)) - - def _matchonekeyword(self, key, chain): - for subitem in chain: - if subitem.haskeyword(key): - return True - return False - - - def _map2colitems(items): - # first convert all path objects into collectors - from py.__.test.collect import getfscollector - colitems = [] - for item in items: - if isinstance(item, (list, tuple)): - colitems.extend(Session._map2colitems(item)) - elif not isinstance(item, py.test.collect.Collector): - colitems.append(getfscollector(item)) - else: - colitems.append(item) - return colitems - _map2colitems = staticmethod(_map2colitems) - -class Exit(Exception): - """ for immediate program exits without tracebacks and reporter/summary. """ - def __init__(self, msg="unknown reason", item=None): - self.msg = msg - Exception.__init__(self, msg) - -def exit(msg, item=None): - raise Exit(msg=msg, item=item) + from py.__.test.terminal.remote import getrootdir + from py.__.test.rsession.master import ( + setup_slave, bin_rsync, MasterNode) + #colitems = self._map2colitems(args) + sshhosts = self.config.getinitialvalue("disthosts") + relpath = "pytestcache" + rootdir = getrootdir(args) + nodes = [] + for host in sshhosts: + bin_rsync(rootdir, host, relpath) + gw = py.execnet.SshGateway(host) + ch = setup_slave(gw, relpath) + nodes.append(MasterNode(ch, self.report)) + print "INITIALIZED, dying now ..." Modified: py/branch/distributed/py/test/testing/test_collect.py ============================================================================== --- py/branch/distributed/py/test/testing/test_collect.py (original) +++ py/branch/distributed/py/test/testing/test_collect.py Tue Aug 22 21:09:39 2006 @@ -34,6 +34,8 @@ assert x.name == "filetest.py" x = col1.getitembynames("/".join(l[1:])) assert x.name == "filetest.py" + l2 = x.listnames() + assert len(l2) == 3 def test_finds_tests(): fn = datadir / 'filetest.py' From fijal at codespeak.net Tue Aug 22 21:35:56 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 Aug 2006 21:35:56 +0200 (CEST) Subject: [py-svn] r31511 - py/branch/distributed/py/test/rsession Message-ID: <20060822193556.1C3FE1005A@code0.codespeak.net> Author: fijal Date: Tue Aug 22 21:35:54 2006 New Revision: 31511 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/rsession.py Log: Added first "working" version of RSession. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Tue Aug 22 21:35:54 2006 @@ -20,7 +20,7 @@ def send(self, item): self.pending.insert(0, item) - itemspec = "/".join(item.listnames()[1:]) + itemspec = "/".join(item.listnames()) self.channel.send(itemspec) def dispatch_loop(masternodes, itemgenerator, shouldstop, event, @@ -37,20 +37,24 @@ break event.wait(3) # XXX argh event.clear() + + # XXX: wait here for completion -_rsync = py.path.local.sysfind("rsync") def bin_rsync(source, sshaddress, destpath): + _rsync = py.path.local.sysfind("rsync") print "rsyncing", source, "to", sshaddress, destpath _rsync.sysexec("-az", "--delete", "--exclude=.svn", str(source), sshaddress + ":" + str(destpath)) def setup_slave(gateway, destpath): - from py.__.test.rsession import slave + from py.__.test.rsession import slave ch = gateway.remote_exec(py.code.Source(slave.setup, "setup()")) ch.send(str(destpath)) - return ch - + return ch + + + #def setup_(gateway, rootdir, report, path="pytest_cache"): # bin_rsync(rootdir, gateway.sshaddress, path) # return MasterNode(ch, report) Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Tue Aug 22 21:35:54 2006 @@ -13,22 +13,43 @@ self.config = config def report(self, *whatever): - print whatever + self.event.set() + print whatever def main(self, args): """ main loop for running tests. """ from py.__.test.terminal.remote import getrootdir from py.__.test.rsession.master import ( - setup_slave, bin_rsync, MasterNode) - #colitems = self._map2colitems(args) + setup_slave, bin_rsync, MasterNode, dispatch_loop) + colitems = self._map2colitems(args) sshhosts = self.config.getinitialvalue("disthosts") relpath = "pytestcache" rootdir = getrootdir(args) nodes = [] + import threading + self.event = threading.Event() for host in sshhosts: bin_rsync(rootdir, host, relpath) gw = py.execnet.SshGateway(host) ch = setup_slave(gw, relpath) nodes.append(MasterNode(ch, self.report)) + itemgenerator = colitems[0].tryiter() + dispatch_loop(nodes, itemgenerator, lambda : False, self.event) print "INITIALIZED, dying now ..." + import time + time.sleep(3) + + def _map2colitems(items): + # first convert all path objects into collectors + from py.__.test.collect import getfscollector + colitems = [] + for item in items: + if isinstance(item, (list, tuple)): + colitems.extend(Session._map2colitems(item)) + elif not isinstance(item, py.test.collect.Collector): + colitems.append(getfscollector(item)) + else: + colitems.append(item) + return colitems + _map2colitems = staticmethod(_map2colitems) From fijal at codespeak.net Wed Aug 23 11:45:53 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 Aug 2006 11:45:53 +0200 (CEST) Subject: [py-svn] r31518 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823094553.4B16310050@code0.codespeak.net> Author: fijal Date: Wed Aug 23 11:45:49 2006 New Revision: 31518 Modified: py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_dispatcher.py py/branch/distributed/py/test/rsession/testing/test_executor.py py/branch/distributed/py/test/rsession/testing/test_slave.py Log: Fixed exception handling to catch everything. RSync run in paraller now. Test passes in a very obscure way. Some minor bugfixes. Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Wed Aug 23 11:45:49 2006 @@ -13,7 +13,7 @@ def _execute(self, fun): try: fun() - except Exception, e: + except: excinfo = py.code.ExceptionInfo() code = py.code.Code(fun) excinfo.traceback = excinfo.traceback.cut( @@ -28,3 +28,17 @@ if excinfo is not None: return Outcome(excinfo=excinfo, setupfailure=False) return Outcome() + +class RunExecutor(Executor): + """ Same as in executor, but just running run + """ + def __init__(self, item): + self.item = item + + def execute(self): + try: + self.item.run() + return Outcome() + except: + excinfo = py.code.ExceptionInfo() + return Outcome(excinfo=excinfo, setupfailure=False) Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 11:45:49 2006 @@ -15,9 +15,24 @@ def report(self, *whatever): self.event.set() print whatever + + def iteritems(self, colitems): + to_run = colitems + try: + while 1: + colitem = to_run.pop() + if isinstance(colitem, py.test.Item): + yield colitem + else: + for i in colitem.run(): + to_run.append(colitem.join(i)) + except IndexError: + pass + - def main(self, args): + def main(self, args): """ main loop for running tests. """ + import thread, threading from py.__.test.terminal.remote import getrootdir from py.__.test.rsession.master import ( setup_slave, bin_rsync, MasterNode, dispatch_loop) @@ -28,17 +43,38 @@ nodes = [] import threading self.event = threading.Event() - for host in sshhosts: + nodes_lock = threading.Condition() + + def host_init(host): bin_rsync(rootdir, host, relpath) gw = py.execnet.SshGateway(host) - ch = setup_slave(gw, relpath) - nodes.append(MasterNode(ch, self.report)) + ch = setup_slave(gw, relpath) + nodes_lock.acquire() + try: + nodes.append(MasterNode(ch, self.report)) + nodes_lock.notify() + finally: + nodes_lock.release() + + for host in sshhosts: + thread.start_new_thread(host_init, (host,)) + + while len(nodes) != len(sshhosts): + nodes_lock.acquire() + try: + nodes_lock.wait(1) + finally: + nodes_lock.release() - itemgenerator = colitems[0].tryiter() + itemgenerator = self.iteritems(colitems) dispatch_loop(nodes, itemgenerator, lambda : False, self.event) print "INITIALIZED, dying now ..." import time time.sleep(3) + + # XXX: Some cleanup here, like counting all the stuff we send + # and waiting for them to arrive back + # Maybe rescheduling stuff that did not landed at all def _map2colitems(items): # first convert all path objects into collectors Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Wed Aug 23 11:45:49 2006 @@ -3,15 +3,18 @@ """ import py -from py.__.test.rsession.executor import Executor +from py.__.test.rsession.executor import Executor, RunExecutor class SlaveNode(object): def __init__(self, rootcollector): self.rootcollector = rootcollector - def execute(self, itemspec): + def execute(self, itemspec): item = self.rootcollector.getitembynames(itemspec) - ex = Executor(item.obj, setup=item.setup) + if isinstance(item, py.test.Function): + ex = Executor(item.obj, setup=item.setup) + else: + ex = RunExecutor(item) return ex.execute() def run(self, itemspec): Modified: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_dispatcher.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Wed Aug 23 11:45:49 2006 @@ -69,6 +69,9 @@ rootcol = py.test.collect.Directory(rootdir) funcpass_item = rootcol.getitembynames(funcpass_spec) funcfail_item = rootcol.getitembynames(funcfail_spec) + # XXX: Just make this test pass + funcpass_item.parent.parent.parent.parent.parent.parent = None + funcfail_item.parent.parent.parent.parent.parent.parent = None itemgenerator = iter([funcfail_item] + [funcpass_item] * 5 + [funcfail_item] * 5) shouldstop = lambda : False dispatch_loop(master_nodes, itemgenerator, shouldstop, event) Modified: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_executor.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Wed Aug 23 11:45:49 2006 @@ -2,7 +2,7 @@ import py import example1 -from py.__.test.rsession.executor import Executor +from py.__.test.rsession.executor import Executor, RunExecutor def test_executor_passing_function(): ex = Executor(example1.f1) @@ -38,3 +38,20 @@ assert not outcome.passed assert outcome.setupfailure assert outcome.excinfo.traceback[-1].frame.code.name == 'failingsetup' + +class ItemTestPassing(py.test.Item): + def run(self): + return None + +class ItemTestFailing(py.test.Item): + def run(self): + assert 0 == 1 + +def test_run_executor(): + ex = RunExecutor(ItemTestPassing("pass")) + outcome = ex.execute() + assert outcome.passed + + ex = RunExecutor(ItemTestFailing("fail")) + outcome = ex.execute() + assert not outcome.passed Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Wed Aug 23 11:45:49 2006 @@ -70,4 +70,6 @@ assert not res_repr[0].passed assert res_repr[1].passed - +def test_slave_run_different_stuff(): + node = gettestnode() + node.run("py/documentation/log.txt") From hpk at codespeak.net Wed Aug 23 12:08:04 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 12:08:04 +0200 (CEST) Subject: [py-svn] r31519 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823100804.0940710050@code0.codespeak.net> Author: hpk Date: Wed Aug 23 12:07:55 2006 New Revision: 31519 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/testing/test_slave.py Log: (fijal, hpk) itemspecs are now sequences Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 12:07:55 2006 @@ -20,7 +20,7 @@ def send(self, item): self.pending.insert(0, item) - itemspec = "/".join(item.listnames()) + itemspec = item.listnames() self.channel.send(itemspec) def dispatch_loop(masternodes, itemgenerator, shouldstop, event, Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Wed Aug 23 12:07:55 2006 @@ -15,8 +15,8 @@ def funcfail(): raise AssertionError("hello world") -funcpass_spec = "py/test/rsession/testing/test_slave.py/funcpass" -funcfail_spec = "py/test/rsession/testing/test_slave.py/funcfail" +funcpass_spec = "py/test/rsession/testing/test_slave.py/funcpass".split("/") +funcfail_spec = "py/test/rsession/testing/test_slave.py/funcfail".split("/") # ---------------------------------------------------------------------- @@ -72,4 +72,4 @@ def test_slave_run_different_stuff(): node = gettestnode() - node.run("py/documentation/log.txt") + node.run("py documentation log.txt".split()) From hpk at codespeak.net Wed Aug 23 12:14:46 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 12:14:46 +0200 (CEST) Subject: [py-svn] r31521 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823101446.0536D10050@code0.codespeak.net> Author: hpk Date: Wed Aug 23 12:14:32 2006 New Revision: 31521 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Log: (fijal, hpk) Remove Event logic, just have an optional waiter function. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 12:14:32 2006 @@ -23,7 +23,8 @@ itemspec = item.listnames() self.channel.send(itemspec) -def dispatch_loop(masternodes, itemgenerator, shouldstop, event, +def dispatch_loop(masternodes, itemgenerator, shouldstop, + waiter = lambda: py.std.time.sleep(0.1), maxtasks_per_node = MAX_TASKS_PER_NODE): while 1: try: @@ -35,12 +36,8 @@ node.send(item) except StopIteration: break - event.wait(3) # XXX argh - event.clear() + waiter() - # XXX: wait here for completion - - def bin_rsync(source, sshaddress, destpath): _rsync = py.path.local.sysfind("rsync") print "rsyncing", source, "to", sshaddress, destpath Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 12:14:32 2006 @@ -13,7 +13,6 @@ self.config = config def report(self, *whatever): - self.event.set() print whatever def iteritems(self, colitems): @@ -42,7 +41,6 @@ rootdir = getrootdir(args) nodes = [] import threading - self.event = threading.Event() nodes_lock = threading.Condition() def host_init(host): @@ -67,7 +65,7 @@ nodes_lock.release() itemgenerator = self.iteritems(colitems) - dispatch_loop(nodes, itemgenerator, lambda : False, self.event) + dispatch_loop(nodes, itemgenerator, lambda : False) print "INITIALIZED, dying now ..." import time time.sleep(3) Modified: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_dispatcher.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Wed Aug 23 12:14:32 2006 @@ -20,23 +20,14 @@ def send(self, data): self.pending.append(data) -class DummyEvent(object): - def __init__(self, nodes): - self.nodes = nodes - - def wait(self, timeout = None): - for node in self.nodes: - node.pending.pop() - - def clear(self): - pass - def test_dispatch_loop(): masternodes = [DummyMasterNode(), DummyMasterNode()] - event = DummyEvent(masternodes) itemgenerator = iter(range(100)) shouldstop = lambda : False - dispatch_loop(masternodes, itemgenerator, shouldstop, event) + def waiter(): + for node in masternodes: + node.pending.pop() + dispatch_loop(masternodes, itemgenerator, shouldstop, waiter=waiter) def test_slave_setup(): gw = py.execnet.PopenGateway() @@ -49,10 +40,7 @@ gw.exit() def test_slave_running(): - event = threading.Event() - def simple_reporter(result): - event.set() item, res = result if item.code.name == 'funcpass': assert res.passed @@ -74,4 +62,4 @@ funcfail_item.parent.parent.parent.parent.parent.parent = None itemgenerator = iter([funcfail_item] + [funcpass_item] * 5 + [funcfail_item] * 5) shouldstop = lambda : False - dispatch_loop(master_nodes, itemgenerator, shouldstop, event) + dispatch_loop(master_nodes, itemgenerator, shouldstop) From hpk at codespeak.net Wed Aug 23 12:34:30 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 12:34:30 +0200 (CEST) Subject: [py-svn] r31523 - in py/branch/distributed/py/test: . testing Message-ID: <20060823103430.E393410050@code0.codespeak.net> Author: hpk Date: Wed Aug 23 12:34:27 2006 New Revision: 31523 Modified: py/branch/distributed/py/test/collect.py py/branch/distributed/py/test/testing/test_collect.py Log: (fijal, hpk) fix tryiter and test it (ugh) Modified: py/branch/distributed/py/test/collect.py ============================================================================== --- py/branch/distributed/py/test/collect.py (original) +++ py/branch/distributed/py/test/collect.py Wed Aug 23 12:34:27 2006 @@ -176,17 +176,21 @@ newl.append(x.name) return ".".join(newl) - def tryiter(self, stopitems=None): + def tryiter(self, yieldtype=None): """ yield stop item instances from flattening the collector. + XXX deprecated: this way of iteration is not safe in all + cases. """ - if stopitems is None: - stopitems = py.test.Item - if isinstance(self, stopitems): + + if yieldtype is None: + yieldtype = py.test.Item + if isinstance(self, yieldtype): yield self else: - for x in self.run(): - for y in self.join(x).tryiter(stopitems): - yield y + if not isinstance(self, py.test.Item): + for x in self.run(): + for y in self.join(x).tryiter(yieldtype): + yield y def _prepare(self): if not hasattr(self, '_name2items'): Modified: py/branch/distributed/py/test/testing/test_collect.py ============================================================================== --- py/branch/distributed/py/test/testing/test_collect.py (original) +++ py/branch/distributed/py/test/testing/test_collect.py Wed Aug 23 12:34:27 2006 @@ -294,7 +294,6 @@ yield assert_order_of_execution - def test_order_of_execution_generator_different_codeline(): test_list = [] expected_list = range(3) @@ -318,3 +317,16 @@ yield list_append_2 yield assert_order_of_execution + +def test_documentation_virtual_collector_interaction(): + rootdir = py.path.local(py.__file__).dirpath("documentation") + # HACK + from py.__.documentation import conftest as conf + old = conf.option.forcegen + try: + conf.option.forcegen = 1 + col = py.test.collect.Directory(rootdir) + x = list(col.tryiter(yieldtype=py.test.Function)) + finally: + conf.option.forcegen = old + From hpk at codespeak.net Wed Aug 23 12:54:45 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 12:54:45 +0200 (CEST) Subject: [py-svn] r31526 - in py/branch/distributed/py/test: . testing Message-ID: <20060823105445.71F5B10069@code0.codespeak.net> Author: hpk Date: Wed Aug 23 12:54:42 2006 New Revision: 31526 Modified: py/branch/distributed/py/test/collect.py py/branch/distributed/py/test/testing/test_collect.py Log: (fijal, hpk) made tryiter to ignore py.test.skip's Modified: py/branch/distributed/py/test/collect.py ============================================================================== --- py/branch/distributed/py/test/collect.py (original) +++ py/branch/distributed/py/test/collect.py Wed Aug 23 12:54:42 2006 @@ -188,9 +188,12 @@ yield self else: if not isinstance(self, py.test.Item): - for x in self.run(): - for y in self.join(x).tryiter(yieldtype): - yield y + try: + for x in self.run(): + for y in self.join(x).tryiter(yieldtype): + yield y + except py.test.Item.Skipped: + pass def _prepare(self): if not hasattr(self, '_name2items'): Modified: py/branch/distributed/py/test/testing/test_collect.py ============================================================================== --- py/branch/distributed/py/test/testing/test_collect.py (original) +++ py/branch/distributed/py/test/testing/test_collect.py Wed Aug 23 12:54:42 2006 @@ -330,3 +330,25 @@ finally: conf.option.forcegen = old + +def test_tryiter_ignores_skips(): + tmp = py.test.ensuretemp("tryiterskip") + tmp.ensure("subdir", "conftest.py").write(py.code.Source(""" + import py + class Directory(py.test.collect.Directory): + def run(self): + py.test.skip("intentional") + """)) + col = py.test.collect.Directory(tmp) + try: + list(col.tryiter()) + except KeyboardInterrupt: + raise + except: + exc = py.code.ExceptionInfo() + py.test.fail("should not have raised: %s" %(exc,)) + + + + + From hpk at codespeak.net Wed Aug 23 13:13:09 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 13:13:09 +0200 (CEST) Subject: [py-svn] r31527 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823111309.7C92C10060@code0.codespeak.net> Author: hpk Date: Wed Aug 23 13:13:02 2006 New Revision: 31527 Modified: py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_executor.py Log: (fijal, hpk) executor groks skipped tests and does not use setup-logic for now. Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Wed Aug 23 13:13:02 2006 @@ -39,6 +39,8 @@ try: self.item.run() return Outcome() + except py.test.Item.Skipped: + return Outcome(skipped=True) except: excinfo = py.code.ExceptionInfo() return Outcome(excinfo=excinfo, setupfailure=False) Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Wed Aug 23 13:13:02 2006 @@ -3,10 +3,12 @@ """ class Outcome(object): - def __init__(self, setupfailure=False, excinfo=None): - self.passed = not excinfo + def __init__(self, setupfailure=False, excinfo=None, skipped=False): + self.passed = not excinfo and not skipped + self.skipped = skipped self.setupfailure = setupfailure self.excinfo = excinfo + assert bool(self.passed) + bool(excinfo) + bool(skipped) == 1 def make_repr(self): return (self.passed, self.setupfailure, repr(self.excinfo)) Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Wed Aug 23 13:13:02 2006 @@ -11,10 +11,10 @@ def execute(self, itemspec): item = self.rootcollector.getitembynames(itemspec) - if isinstance(item, py.test.Function): - ex = Executor(item.obj, setup=item.setup) - else: - ex = RunExecutor(item) + #if isinstance(item, py.test.Function): + # ex = Executor(item.obj, setup=item.setup) + #else: + ex = RunExecutor(item) return ex.execute() def run(self, itemspec): Modified: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_executor.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Wed Aug 23 13:13:02 2006 @@ -24,13 +24,13 @@ assert excinfo.traceback[1].frame.code.name == 'g2' assert excinfo.traceback[0].frame.code.name == 'g1' -def test_executor_setup_passing(): +def XXX_test_executor_setup_passing(): ex = Executor(example1.f1, setup=lambda: None) outcome = ex.execute() assert outcome.passed assert not outcome.setupfailure -def test_executor_setup_failing(): +def XXX_test_executor_setup_failing(): def failingsetup(): raise ValueError ex = Executor(example1.f1, setup=failingsetup) @@ -47,6 +47,10 @@ def run(self): assert 0 == 1 +class ItemTestSkipping(py.test.Item): + def run(self): + py.test.skip("hello") + def test_run_executor(): ex = RunExecutor(ItemTestPassing("pass")) outcome = ex.execute() @@ -55,3 +59,9 @@ ex = RunExecutor(ItemTestFailing("fail")) outcome = ex.execute() assert not outcome.passed + + ex = RunExecutor(ItemTestSkipping("fail")) + outcome = ex.execute() + assert outcome.skipped + assert not outcome.passed + assert not outcome.excinfo From hpk at codespeak.net Wed Aug 23 13:34:02 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 13:34:02 +0200 (CEST) Subject: [py-svn] r31533 - in py/branch/distributed/py/test: . testing Message-ID: <20060823113402.9F2CB10034@code0.codespeak.net> Author: hpk Date: Wed Aug 23 13:34:00 2006 New Revision: 31533 Modified: py/branch/distributed/py/test/collect.py py/branch/distributed/py/test/testing/test_collect.py Log: (fijal, hpk) tryiter by default ignores col.run() failures or alternatively reports to a given reporting functions. Modified: py/branch/distributed/py/test/collect.py ============================================================================== --- py/branch/distributed/py/test/collect.py (original) +++ py/branch/distributed/py/test/collect.py Wed Aug 23 13:34:00 2006 @@ -176,7 +176,7 @@ newl.append(x.name) return ".".join(newl) - def tryiter(self, yieldtype=None): + def tryiter(self, yieldtype=None, reporterror=None): """ yield stop item instances from flattening the collector. XXX deprecated: this way of iteration is not safe in all cases. @@ -192,8 +192,12 @@ for x in self.run(): for y in self.join(x).tryiter(yieldtype): yield y - except py.test.Item.Skipped: - pass + except KeyboardInterrupt: + raise + except: + if reporterror is not None: + excinfo = py.code.ExceptionInfo() + reporterror(excinfo) def _prepare(self): if not hasattr(self, '_name2items'): Modified: py/branch/distributed/py/test/testing/test_collect.py ============================================================================== --- py/branch/distributed/py/test/testing/test_collect.py (original) +++ py/branch/distributed/py/test/testing/test_collect.py Wed Aug 23 13:34:00 2006 @@ -349,6 +349,21 @@ py.test.fail("should not have raised: %s" %(exc,)) +def test_tryiter_ignores_failing_collectors(): + tmp = py.test.ensuretemp("tryiterfailing") + tmp.ensure("subdir", "conftest.py").write(py.code.Source(""" + bla bla bla + """)) + col = py.test.collect.Directory(tmp) + try: + list(col.tryiter()) + except KeyboardInterrupt: + raise + except: + exc = py.code.ExceptionInfo() + py.test.fail("should not have raised: %s" %(exc,)) - - + l = [] + list(col.tryiter(reporterror=l.append)) + assert len(l) == 1 + assert isinstance(l[0], py.code.ExceptionInfo) From hpk at codespeak.net Wed Aug 23 13:39:44 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 13:39:44 +0200 (CEST) Subject: [py-svn] r31536 - py/branch/distributed/py/test/testing Message-ID: <20060823113944.0691410060@code0.codespeak.net> Author: hpk Date: Wed Aug 23 13:39:41 2006 New Revision: 31536 Modified: py/branch/distributed/py/test/testing/test_collect.py Log: (fijal, hpk) 99% test for keyboardinterrupts Modified: py/branch/distributed/py/test/testing/test_collect.py ============================================================================== --- py/branch/distributed/py/test/testing/test_collect.py (original) +++ py/branch/distributed/py/test/testing/test_collect.py Wed Aug 23 13:39:41 2006 @@ -367,3 +367,11 @@ list(col.tryiter(reporterror=l.append)) assert len(l) == 1 assert isinstance(l[0], py.code.ExceptionInfo) + +def test_tryiter_handles_keyboardinterrupt(): + tmp = py.test.ensuretemp("tryiterkeyboard") + tmp.ensure("subdir", "conftest.py").write(py.code.Source(""" + raise KeyboardInterrupt() + """)) + col = py.test.collect.Directory(tmp) + py.test.raises(KeyboardInterrupt, list, col.tryiter()) From hpk at codespeak.net Wed Aug 23 13:46:21 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 13:46:21 +0200 (CEST) Subject: [py-svn] r31538 - py/branch/distributed/py/test/rsession Message-ID: <20060823114621.6660F10036@code0.codespeak.net> Author: hpk Date: Wed Aug 23 13:46:19 2006 New Revision: 31538 Modified: py/branch/distributed/py/test/rsession/executor.py Log: (fijal, hpk) fix exception cutting Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Wed Aug 23 13:46:19 2006 @@ -43,4 +43,9 @@ return Outcome(skipped=True) except: excinfo = py.code.ExceptionInfo() + if isinstance(self.item, py.test.Function): + fun = self.item.obj # hope this is stable + code = py.code.Code(fun) + excinfo.traceback = excinfo.traceback.cut( + path=code.path, firstlineno=code.firstlineno) return Outcome(excinfo=excinfo, setupfailure=False) From fijal at codespeak.net Wed Aug 23 13:47:05 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 Aug 2006 13:47:05 +0200 (CEST) Subject: [py-svn] r31539 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823114705.58AB010036@code0.codespeak.net> Author: fijal Date: Wed Aug 23 13:47:02 2006 New Revision: 31539 Modified: py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_slave.py Log: (hpk, fijal) - added skipped flag serialisation. Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Wed Aug 23 13:47:02 2006 @@ -11,8 +11,8 @@ assert bool(self.passed) + bool(excinfo) + bool(skipped) == 1 def make_repr(self): - return (self.passed, self.setupfailure, repr(self.excinfo)) + return (self.passed, self.setupfailure, repr(self.excinfo), self.skipped) class ReprOutcome(object): def __init__(self, repr_tuple): - self.passed, self.setupfailure, self.excinfo = repr_tuple + self.passed, self.setupfailure, self.excinfo, self.skipped = repr_tuple Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 13:47:02 2006 @@ -15,20 +15,6 @@ def report(self, *whatever): print whatever - def iteritems(self, colitems): - to_run = colitems - try: - while 1: - colitem = to_run.pop() - if isinstance(colitem, py.test.Item): - yield colitem - else: - for i in colitem.run(): - to_run.append(colitem.join(i)) - except IndexError: - pass - - def main(self, args): """ main loop for running tests. """ import thread, threading @@ -64,7 +50,7 @@ finally: nodes_lock.release() - itemgenerator = self.iteritems(colitems) + itemgenerator = colitems[0].tryiter() dispatch_loop(nodes, itemgenerator, lambda : False) print "INITIALIZED, dying now ..." import time Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Wed Aug 23 13:47:02 2006 @@ -15,8 +15,13 @@ def funcfail(): raise AssertionError("hello world") +def funcskip(): + py.test.skip("skipped") + funcpass_spec = "py/test/rsession/testing/test_slave.py/funcpass".split("/") funcfail_spec = "py/test/rsession/testing/test_slave.py/funcfail".split("/") +funcskip_spec = "py/test/rsession/testing/test_slave.py/funcskip".split("/") + # ---------------------------------------------------------------------- @@ -49,6 +54,17 @@ assert not reproutcome.passed assert not reproutcome.setupfailure assert reproutcome.excinfo + +def test_slave_run_skipping(): + node = gettestnode() + outcome = node.execute(funcskip_spec) + assert not outcome.passed + assert outcome.skipped + + ser = outcome.make_repr() + reproutcome = ReprOutcome(ser) + assert not reproutcome.passed + assert reproutcome.skipped def test_slave_run_failing_wrapped(): node = gettestnode() From hpk at codespeak.net Wed Aug 23 14:33:51 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 14:33:51 +0200 (CEST) Subject: [py-svn] r31540 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060823123351.4F9E710050@code0.codespeak.net> Author: hpk Date: Wed Aug 23 14:33:48 2006 New Revision: 31540 Removed: py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Modified: py/branch/distributed/py/test/rsession/testing/test_master.py Log: merge tests Deleted: /py/branch/distributed/py/test/rsession/testing/test_dispatcher.py ============================================================================== --- /py/branch/distributed/py/test/rsession/testing/test_dispatcher.py Wed Aug 23 14:33:48 2006 +++ (empty file) @@ -1,65 +0,0 @@ - -""" test dispatching stuff for making 1-n master -> slave connection -and test it locally -""" - -import time, threading - -import py -from py.__.test.rsession.master import dispatch_loop, setup_slave, MasterNode -from py.__.test.rsession.outcome import ReprOutcome -from py.__.test.rsession.testing.test_slave import funcpass_spec, funcfail_spec - -def setup_module(mod): - mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() - -class DummyMasterNode(object): - def __init__(self): - self.pending = [] - - def send(self, data): - self.pending.append(data) - -def test_dispatch_loop(): - masternodes = [DummyMasterNode(), DummyMasterNode()] - itemgenerator = iter(range(100)) - shouldstop = lambda : False - def waiter(): - for node in masternodes: - node.pending.pop() - dispatch_loop(masternodes, itemgenerator, shouldstop, waiter=waiter) - -def test_slave_setup(): - gw = py.execnet.PopenGateway() - channel = setup_slave(gw, rootdir) - channel.send(funcpass_spec) - output = ReprOutcome(channel.receive()) - assert output.passed - channel.send(None) - channel.waitclose(10) - gw.exit() - -def test_slave_running(): - def simple_reporter(result): - item, res = result - if item.code.name == 'funcpass': - assert res.passed - else: - assert not res.passed - - def open_gw(): - gw = py.execnet.PopenGateway() - channel = setup_slave(gw, rootdir) - mn = MasterNode(channel, simple_reporter) - return mn - - master_nodes = [open_gw(), open_gw(), open_gw()] - rootcol = py.test.collect.Directory(rootdir) - funcpass_item = rootcol.getitembynames(funcpass_spec) - funcfail_item = rootcol.getitembynames(funcfail_spec) - # XXX: Just make this test pass - funcpass_item.parent.parent.parent.parent.parent.parent = None - funcfail_item.parent.parent.parent.parent.parent.parent = None - itemgenerator = iter([funcfail_item] + [funcpass_item] * 5 + [funcfail_item] * 5) - shouldstop = lambda : False - dispatch_loop(master_nodes, itemgenerator, shouldstop) Modified: py/branch/distributed/py/test/rsession/testing/test_master.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_master.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Wed Aug 23 14:33:48 2006 @@ -1,8 +1,16 @@ -""" test master node """ +""" master code and test dispatching for + making 1-n master -> slave connection + and test it locally. +""" +import time, threading import py -from py.__.test.rsession.master import MasterNode -from py.__.test.rsession.outcome import Outcome +from py.__.test.rsession.master import dispatch_loop, setup_slave, MasterNode +from py.__.test.rsession.outcome import ReprOutcome, Outcome +from py.__.test.rsession.testing.test_slave import funcpass_spec, funcfail_spec + +def setup_module(mod): + mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() class DummyChannel(object): def __init__(self): @@ -26,3 +34,55 @@ assert len(reportlist) == 2 assert reportlist[0][1].passed assert not reportlist[1][1].passed + + +class DummyMasterNode(object): + def __init__(self): + self.pending = [] + + def send(self, data): + self.pending.append(data) + +def test_dispatch_loop(): + masternodes = [DummyMasterNode(), DummyMasterNode()] + itemgenerator = iter(range(100)) + shouldstop = lambda : False + def waiter(): + for node in masternodes: + node.pending.pop() + dispatch_loop(masternodes, itemgenerator, shouldstop, waiter=waiter) + +def test_slave_setup(): + gw = py.execnet.PopenGateway() + channel = setup_slave(gw, rootdir) + channel.send(funcpass_spec) + output = ReprOutcome(channel.receive()) + assert output.passed + channel.send(None) + channel.waitclose(10) + gw.exit() + +def test_slave_running(): + def simple_reporter(result): + item, res = result + if item.code.name == 'funcpass': + assert res.passed + else: + assert not res.passed + + def open_gw(): + gw = py.execnet.PopenGateway() + channel = setup_slave(gw, rootdir) + mn = MasterNode(channel, simple_reporter) + return mn + + master_nodes = [open_gw(), open_gw(), open_gw()] + rootcol = py.test.collect.Directory(rootdir) + funcpass_item = rootcol.getitembynames(funcpass_spec) + funcfail_item = rootcol.getitembynames(funcfail_spec) + # XXX: Just make this test pass + funcpass_item.parent.parent.parent.parent.parent.parent = None + funcfail_item.parent.parent.parent.parent.parent.parent = None + itemgenerator = iter([funcfail_item] + [funcpass_item] * 5 + [funcfail_item] * 5) + shouldstop = lambda : False + dispatch_loop(master_nodes, itemgenerator, shouldstop) From hpk at codespeak.net Wed Aug 23 14:40:51 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 14:40:51 +0200 (CEST) Subject: [py-svn] r31541 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823124051.B3D1D10050@code0.codespeak.net> Author: hpk Date: Wed Aug 23 14:40:48 2006 New Revision: 31541 Modified: py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/testing/test_master.py Log: nicer ReprOutcome repr Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Wed Aug 23 14:40:48 2006 @@ -16,3 +16,7 @@ class ReprOutcome(object): def __init__(self, repr_tuple): self.passed, self.setupfailure, self.excinfo, self.skipped = repr_tuple + def __repr__(self): + l = ["%s=%s" %(x, getattr(self, x)) + for x in "passed skipped setupfailure excinfo".split()] + return "" %(" ".join(l),) Modified: py/branch/distributed/py/test/rsession/testing/test_master.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_master.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Wed Aug 23 14:40:48 2006 @@ -35,6 +35,10 @@ assert reportlist[0][1].passed assert not reportlist[1][1].passed +def test_outcome_repr(): + out = ReprOutcome(Outcome(skipped=True).make_repr()) + s = repr(out) + assert s.lower().find("skip") != -1 class DummyMasterNode(object): def __init__(self): From fijal at codespeak.net Wed Aug 23 15:00:42 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 Aug 2006 15:00:42 +0200 (CEST) Subject: [py-svn] r31542 - in py/branch/distributed/py/test: . rsession rsession/testing Message-ID: <20060823130042.8A58C10060@code0.codespeak.net> Author: fijal Date: Wed Aug 23 15:00:38 2006 New Revision: 31542 Added: py/branch/distributed/py/test/rsession/reporter.py (contents, props changed) Modified: py/branch/distributed/py/test/collect.py py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_master.py Log: (hpk, fijal) - Added simple reporter. Modified: py/branch/distributed/py/test/collect.py ============================================================================== --- py/branch/distributed/py/test/collect.py (original) +++ py/branch/distributed/py/test/collect.py Wed Aug 23 15:00:38 2006 @@ -153,7 +153,7 @@ cur = self for name in namelist: next = cur.join(name) - assert next is not None, (cur, name) + assert next is not None, (cur, name, namelist) cur = next return cur Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 15:00:38 2006 @@ -2,7 +2,8 @@ Node code for Master. """ import py -from py.__.test.rsession.outcome import ReprOutcome +from py.__.test.rsession.outcome import ReprOutcome +from py.__.test.rsession import reporter MAX_TASKS_PER_NODE = 5 @@ -16,12 +17,15 @@ def receive_result(self, outcomestring): repr_outcome = ReprOutcome(outcomestring) item = self.pending.pop() - self.report((item, repr_outcome)) + # send finish report + self.report(reporter.ReceivedItemOutcome(self.channel, item, repr_outcome)) def send(self, item): self.pending.insert(0, item) - itemspec = item.listnames() + itemspec = item.listnames() self.channel.send(itemspec) + # send start report + self.report(reporter.SendItem(self.channel, item)) def dispatch_loop(masternodes, itemgenerator, shouldstop, waiter = lambda: py.std.time.sleep(0.1), Added: py/branch/distributed/py/test/rsession/reporter.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/reporter.py Wed Aug 23 15:00:38 2006 @@ -0,0 +1,38 @@ + +""" Reporter classes for showing asynchronous and synchronous status events +""" + +import py + +def basic_report(msg_type, message): + print msg_type, message + +#def report(msg_type, message): +# pass + +def report_error(excinfo): + if isinstance(excinfo, py.test.Item.Skipped): + # we need to dispatch this info + report("skipped", excinfo) + else: + report("itererror", excinfo) + +class ReportEvent(object): + def __repr__(self): + l = ["%s=%s" %(key, value) + for key, value in self.__dict__.items()] + return "<%s %s>" %(self.__class__.__name__, " ".join(l),) + +class SendItem(ReportEvent): + def __init__(self, channel, item): + self.item = item + self.channel = channel + +class ReceivedItemOutcome(ReportEvent): + def __init__(self, channel, item, outcome): + self.channel = channel + self.item = item + self.outcome = outcome + +# tryiter, main dispatch loop, something else (setup-teardown stuff) +# Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 15:00:38 2006 @@ -1,4 +1,5 @@ import py +import threading class Reporter(object): def __init__(self, config): @@ -15,18 +16,8 @@ def report(self, *whatever): print whatever - def main(self, args): - """ main loop for running tests. """ - import thread, threading - from py.__.test.terminal.remote import getrootdir - from py.__.test.rsession.master import ( - setup_slave, bin_rsync, MasterNode, dispatch_loop) - colitems = self._map2colitems(args) - sshhosts = self.config.getinitialvalue("disthosts") - relpath = "pytestcache" - rootdir = getrootdir(args) + def init_hosts(self, sshhosts, relpath): nodes = [] - import threading nodes_lock = threading.Condition() def host_init(host): @@ -49,6 +40,21 @@ nodes_lock.wait(1) finally: nodes_lock.release() + + return nodes + + + def main(self, args): + """ main loop for running tests. """ + import thread, threading + from py.__.test.terminal.remote import getrootdir + from py.__.test.rsession.master import ( + setup_slave, bin_rsync, MasterNode, dispatch_loop) + colitems = self._map2colitems(args) + sshhosts = self.config.getinitialvalue("disthosts") + relpath = "pytestcache" + rootdir = getrootdir(args) + nodes = self.init_hosts(sshhosts, relpath) itemgenerator = colitems[0].tryiter() dispatch_loop(nodes, itemgenerator, lambda : False) Modified: py/branch/distributed/py/test/rsession/testing/test_master.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_master.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Wed Aug 23 15:00:38 2006 @@ -8,6 +8,7 @@ from py.__.test.rsession.master import dispatch_loop, setup_slave, MasterNode from py.__.test.rsession.outcome import ReprOutcome, Outcome from py.__.test.rsession.testing.test_slave import funcpass_spec, funcfail_spec +from py.__.test.rsession import reporter def setup_module(mod): mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() @@ -31,9 +32,11 @@ mnode.send(py.test.Item("notok")) ch.callback(Outcome().make_repr()) ch.callback(Outcome(excinfo=42).make_repr()) - assert len(reportlist) == 2 - assert reportlist[0][1].passed - assert not reportlist[1][1].passed + assert len(reportlist) == 4 + received = [i for i in reportlist + if isinstance(i, reporter.ReceivedItemOutcome)] + assert received[0].outcome.passed + assert not received[1].outcome.passed def test_outcome_repr(): out = ReprOutcome(Outcome(skipped=True).make_repr()) @@ -67,12 +70,14 @@ gw.exit() def test_slave_running(): - def simple_reporter(result): - item, res = result + def simple_reporter(event): + if not isinstance(event, reporter.ReceivedItemOutcome): + return + item = event.item if item.code.name == 'funcpass': - assert res.passed + assert event.outcome.passed else: - assert not res.passed + assert not event.outcome.passed def open_gw(): gw = py.execnet.PopenGateway() From hpk at codespeak.net Wed Aug 23 15:03:18 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 15:03:18 +0200 (CEST) Subject: [py-svn] r31544 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823130318.DCA3D10060@code0.codespeak.net> Author: hpk Date: Wed Aug 23 15:03:15 2006 New Revision: 31544 Added: py/branch/distributed/py/test/rsession/report.py - copied unchanged from r31542, py/branch/distributed/py/test/rsession/reporter.py Removed: py/branch/distributed/py/test/rsession/reporter.py Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/testing/test_master.py Log: (hpk, fijal) renamiong reporter to report. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 15:03:15 2006 @@ -3,7 +3,7 @@ """ import py from py.__.test.rsession.outcome import ReprOutcome -from py.__.test.rsession import reporter +from py.__.test.rsession import report MAX_TASKS_PER_NODE = 5 @@ -18,14 +18,14 @@ repr_outcome = ReprOutcome(outcomestring) item = self.pending.pop() # send finish report - self.report(reporter.ReceivedItemOutcome(self.channel, item, repr_outcome)) + self.report(report.ReceivedItemOutcome(self.channel, item, repr_outcome)) def send(self, item): self.pending.insert(0, item) itemspec = item.listnames() self.channel.send(itemspec) # send start report - self.report(reporter.SendItem(self.channel, item)) + self.report(report.SendItem(self.channel, item)) def dispatch_loop(masternodes, itemgenerator, shouldstop, waiter = lambda: py.std.time.sleep(0.1), Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Wed Aug 23 15:03:15 2006 @@ -16,6 +16,7 @@ class ReprOutcome(object): def __init__(self, repr_tuple): self.passed, self.setupfailure, self.excinfo, self.skipped = repr_tuple + def __repr__(self): l = ["%s=%s" %(x, getattr(self, x)) for x in "passed skipped setupfailure excinfo".split()] Deleted: /py/branch/distributed/py/test/rsession/reporter.py ============================================================================== --- /py/branch/distributed/py/test/rsession/reporter.py Wed Aug 23 15:03:15 2006 +++ (empty file) @@ -1,38 +0,0 @@ - -""" Reporter classes for showing asynchronous and synchronous status events -""" - -import py - -def basic_report(msg_type, message): - print msg_type, message - -#def report(msg_type, message): -# pass - -def report_error(excinfo): - if isinstance(excinfo, py.test.Item.Skipped): - # we need to dispatch this info - report("skipped", excinfo) - else: - report("itererror", excinfo) - -class ReportEvent(object): - def __repr__(self): - l = ["%s=%s" %(key, value) - for key, value in self.__dict__.items()] - return "<%s %s>" %(self.__class__.__name__, " ".join(l),) - -class SendItem(ReportEvent): - def __init__(self, channel, item): - self.item = item - self.channel = channel - -class ReceivedItemOutcome(ReportEvent): - def __init__(self, channel, item, outcome): - self.channel = channel - self.item = item - self.outcome = outcome - -# tryiter, main dispatch loop, something else (setup-teardown stuff) -# Modified: py/branch/distributed/py/test/rsession/testing/test_master.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_master.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Wed Aug 23 15:03:15 2006 @@ -8,7 +8,7 @@ from py.__.test.rsession.master import dispatch_loop, setup_slave, MasterNode from py.__.test.rsession.outcome import ReprOutcome, Outcome from py.__.test.rsession.testing.test_slave import funcpass_spec, funcfail_spec -from py.__.test.rsession import reporter +from py.__.test.rsession import report def setup_module(mod): mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() @@ -34,7 +34,7 @@ ch.callback(Outcome(excinfo=42).make_repr()) assert len(reportlist) == 4 received = [i for i in reportlist - if isinstance(i, reporter.ReceivedItemOutcome)] + if isinstance(i, report.ReceivedItemOutcome)] assert received[0].outcome.passed assert not received[1].outcome.passed @@ -70,8 +70,8 @@ gw.exit() def test_slave_running(): - def simple_reporter(event): - if not isinstance(event, reporter.ReceivedItemOutcome): + def simple_report(event): + if not isinstance(event, report.ReceivedItemOutcome): return item = event.item if item.code.name == 'funcpass': @@ -82,7 +82,7 @@ def open_gw(): gw = py.execnet.PopenGateway() channel = setup_slave(gw, rootdir) - mn = MasterNode(channel, simple_reporter) + mn = MasterNode(channel, simple_report) return mn master_nodes = [open_gw(), open_gw(), open_gw()] From hpk at codespeak.net Wed Aug 23 15:16:57 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 15:16:57 +0200 (CEST) Subject: [py-svn] r31546 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823131657.323E410068@code0.codespeak.net> Author: hpk Date: Wed Aug 23 15:16:55 2006 New Revision: 31546 Added: py/branch/distributed/py/test/rsession/testing/test_report.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/report.py Log: (fijal, hpk) add simple call report wrapper Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 15:16:55 2006 @@ -41,7 +41,7 @@ except StopIteration: break waiter() - + def bin_rsync(source, sshaddress, destpath): _rsync = py.path.local.sysfind("rsync") print "rsyncing", source, "to", sshaddress, destpath Modified: py/branch/distributed/py/test/rsession/report.py ============================================================================== --- py/branch/distributed/py/test/rsession/report.py (original) +++ py/branch/distributed/py/test/rsession/report.py Wed Aug 23 15:16:55 2006 @@ -17,6 +17,20 @@ else: report("itererror", excinfo) +def wrapcall(reporter, func, *args, **kwargs): + reporter(CallStart(func, args, kwargs)) + try: + return func(*args, **kwargs) + except: + reporter(CallException(func, args, kwargs)) + raise + else: + reporter(CallFinish(func, args, kwargs)) + +# ---------------------------------------------------------------------- +# Reporting Events +# ---------------------------------------------------------------------- + class ReportEvent(object): def __repr__(self): l = ["%s=%s" %(key, value) Added: py/branch/distributed/py/test/rsession/testing/test_report.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_report.py Wed Aug 23 15:16:55 2006 @@ -0,0 +1,14 @@ +""" test reporting functionality. """ + +from py.__.test.rsession import report + +def test_wrapcall_ok(): + l = [] + def ok(x): + return x+1 + i = report.wrapcall(l.append, ok, 1) + assert i == 2 + assert len(l) == 2 + assert isinstance(l[0], report.CallStart) + assert isinstance(l[1], report.CallFinish) + From fijal at codespeak.net Wed Aug 23 15:19:07 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 Aug 2006 15:19:07 +0200 (CEST) Subject: [py-svn] r31547 - py/branch/distributed/py/test/rsession Message-ID: <20060823131907.6F96D10068@code0.codespeak.net> Author: fijal Date: Wed Aug 23 15:19:05 2006 New Revision: 31547 Modified: py/branch/distributed/py/test/rsession/report.py Log: Fixed test. Modified: py/branch/distributed/py/test/rsession/report.py ============================================================================== --- py/branch/distributed/py/test/rsession/report.py (original) +++ py/branch/distributed/py/test/rsession/report.py Wed Aug 23 15:19:05 2006 @@ -10,22 +10,23 @@ #def report(msg_type, message): # pass -def report_error(excinfo): - if isinstance(excinfo, py.test.Item.Skipped): - # we need to dispatch this info - report("skipped", excinfo) - else: - report("itererror", excinfo) +##def report_error(excinfo): +## if isinstance(excinfo, py.test.Item.Skipped): +## # we need to dispatch this info +## report(Skipped(excinfo)) +## else: +## report("itererror", excinfo) def wrapcall(reporter, func, *args, **kwargs): reporter(CallStart(func, args, kwargs)) try: - return func(*args, **kwargs) + retval = func(*args, **kwargs) except: reporter(CallException(func, args, kwargs)) raise else: reporter(CallFinish(func, args, kwargs)) + return retval # ---------------------------------------------------------------------- # Reporting Events @@ -48,5 +49,20 @@ self.item = item self.outcome = outcome +class CallEvent(ReportEvent): + def __init__(self, func, args, kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + +class CallStart(CallEvent): + pass + +class CallException(CallEvent): + pass + +class CallFinish(CallEvent): + pass + # tryiter, main dispatch loop, something else (setup-teardown stuff) # From hpk at codespeak.net Wed Aug 23 15:20:52 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 15:20:52 +0200 (CEST) Subject: [py-svn] r31548 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060823132052.3B2F410060@code0.codespeak.net> Author: hpk Date: Wed Aug 23 15:20:50 2006 New Revision: 31548 Modified: py/branch/distributed/py/test/rsession/testing/test_report.py Log: add failing test Modified: py/branch/distributed/py/test/rsession/testing/test_report.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_report.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_report.py Wed Aug 23 15:20:50 2006 @@ -1,5 +1,6 @@ """ test reporting functionality. """ +import py from py.__.test.rsession import report def test_wrapcall_ok(): @@ -12,3 +13,12 @@ assert isinstance(l[0], report.CallStart) assert isinstance(l[1], report.CallFinish) +def test_wrapcall_exception(): + l = [] + def fail(x): + raise ValueError + py.test.raises(ValueError, "report.wrapcall(l.append, fail, 1)") + assert len(l) == 2 + assert isinstance(l[0], report.CallStart) + assert isinstance(l[1], report.CallException) + From hpk at codespeak.net Wed Aug 23 15:46:30 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 15:46:30 +0200 (CEST) Subject: [py-svn] r31550 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823134630.128D210036@code0.codespeak.net> Author: hpk Date: Wed Aug 23 15:46:26 2006 New Revision: 31550 Modified: py/branch/distributed/py/test/rsession/report.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_report.py Log: cleanup + adding special Call events repr Modified: py/branch/distributed/py/test/rsession/report.py ============================================================================== --- py/branch/distributed/py/test/rsession/report.py (original) +++ py/branch/distributed/py/test/rsession/report.py Wed Aug 23 15:46:26 2006 @@ -54,6 +54,11 @@ self.func = func self.args = args self.kwargs = kwargs + + def __repr__(self): + call = "%s args=%s, kwargs=%s" %(self.func.__name__, + self.args, self.kwargs) + return '<%s %s>' %(self.__class__.__name__, call) class CallStart(CallEvent): pass Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 15:46:26 2006 @@ -1,10 +1,6 @@ import py import threading -class Reporter(object): - def __init__(self, config): - self.config = config - class RSession(object): """ An RSession executes collectors/items through a runner. Modified: py/branch/distributed/py/test/rsession/testing/test_report.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_report.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_report.py Wed Aug 23 15:46:26 2006 @@ -12,6 +12,8 @@ assert len(l) == 2 assert isinstance(l[0], report.CallStart) assert isinstance(l[1], report.CallFinish) + assert repr(l[0]) + assert repr(l[1]) def test_wrapcall_exception(): l = [] From fijal at codespeak.net Wed Aug 23 16:03:51 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 Aug 2006 16:03:51 +0200 (CEST) Subject: [py-svn] r31551 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823140351.9630A10050@code0.codespeak.net> Author: fijal Date: Wed Aug 23 16:03:48 2006 New Revision: 31551 Added: py/branch/distributed/py/test/rsession/testing/test_rsession.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/rsession.py Log: Added setup/teardown that passes. Gateway now exits cleanly. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 16:03:48 2006 @@ -8,9 +8,9 @@ MAX_TASKS_PER_NODE = 5 class MasterNode(object): - def __init__(self, channel, report): + def __init__(self, channel, reporter): self.channel = channel - self.report = report + self.reporter = reporter channel.setcallback(self.receive_result) self.pending = [] @@ -18,14 +18,14 @@ repr_outcome = ReprOutcome(outcomestring) item = self.pending.pop() # send finish report - self.report(report.ReceivedItemOutcome(self.channel, item, repr_outcome)) + self.reporter(report.ReceivedItemOutcome(self.channel, item, repr_outcome)) def send(self, item): self.pending.insert(0, item) itemspec = item.listnames() self.channel.send(itemspec) # send start report - self.report(report.SendItem(self.channel, item)) + self.reporter(report.SendItem(self.channel, item)) def dispatch_loop(masternodes, itemgenerator, shouldstop, waiter = lambda: py.std.time.sleep(0.1), Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 16:03:48 2006 @@ -1,6 +1,52 @@ + +""" Remote session base class +""" + import py +import thread import threading +from py.__.test.rsession import report + +from py.__.test.terminal.remote import getrootdir +from py.__.test.rsession.master import ( + setup_slave, bin_rsync, MasterNode, dispatch_loop) + +def init_hosts(reporter, sshhosts, relpath, rootdir): + nodes = [] + nodes_lock = threading.Condition() + + def host_init(host): + report.wrapcall(reporter, bin_rsync, rootdir, host, relpath) + gw = py.execnet.SshGateway(host) + ch = setup_slave(gw, relpath) + nodes_lock.acquire() + try: + nodes.append(MasterNode(ch, report)) + nodes_lock.notify() + finally: + nodes_lock.release() + + for host in sshhosts: + thread.start_new_thread(host_init, (host,)) + + while len(nodes) != len(sshhosts): + nodes_lock.acquire() + try: + nodes_lock.wait(1) + finally: + nodes_lock.release() + + return nodes + +def teardown_hosts(reporter, channels): + for channel in channels: + channel.send(None) + + for channel in channels: + report.wrapcall(reporter, channel.waitclose) + channel.gateway.exit() + class RSession(object): """ An RSession executes collectors/items through a runner. @@ -12,51 +58,17 @@ def report(self, *whatever): print whatever - def init_hosts(self, sshhosts, relpath): - nodes = [] - nodes_lock = threading.Condition() - - def host_init(host): - bin_rsync(rootdir, host, relpath) - gw = py.execnet.SshGateway(host) - ch = setup_slave(gw, relpath) - nodes_lock.acquire() - try: - nodes.append(MasterNode(ch, self.report)) - nodes_lock.notify() - finally: - nodes_lock.release() - - for host in sshhosts: - thread.start_new_thread(host_init, (host,)) - - while len(nodes) != len(sshhosts): - nodes_lock.acquire() - try: - nodes_lock.wait(1) - finally: - nodes_lock.release() - - return nodes - - def main(self, args): """ main loop for running tests. """ - import thread, threading - from py.__.test.terminal.remote import getrootdir - from py.__.test.rsession.master import ( - setup_slave, bin_rsync, MasterNode, dispatch_loop) colitems = self._map2colitems(args) sshhosts = self.config.getinitialvalue("disthosts") relpath = "pytestcache" rootdir = getrootdir(args) - nodes = self.init_hosts(sshhosts, relpath) + nodes = init_hosts(self.report, sshhosts, relpath, rootdir) itemgenerator = colitems[0].tryiter() dispatch_loop(nodes, itemgenerator, lambda : False) - print "INITIALIZED, dying now ..." - import time - time.sleep(3) + teardown_hosts(self.report, [node.channel for node in nodes]) # XXX: Some cleanup here, like counting all the stuff we send # and waiting for them to arrive back Added: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Wed Aug 23 16:03:48 2006 @@ -0,0 +1,33 @@ + +""" Tests various aspects of rsession, like ssh hosts setup/teardown +""" + +# XXX: let's skip those tests if we do not have a disthots declared + +hosts = ['codespeak.net', 'snake'] + +import py +from py.__.test.rsession import report +from py.__.test.terminal.remote import getrootdir + +def test_setup_teardown_ssh(): + from py.__.test.rsession.rsession import init_hosts, teardown_hosts + + setup_events = [] + teardown_events = [] + + rootdir = str(py.path.local(py.__file__).dirpath()) + + nodes = init_hosts(setup_events.append, hosts, 'pytestest', rootdir) + teardown_hosts(teardown_events.append, [node.channel for node in nodes]) + + count_rsyn_calls = [i for i in setup_events if isinstance(i, report.CallStart)] + assert len(count_rsyn_calls) == len(hosts) + count_rsyn_ends = [i for i in setup_events if isinstance(i, report.CallFinish)] + assert len(count_rsyn_ends) == len(hosts) + + # same for teardown events + teardown_wait_starts = [i for i in teardown_events if isinstance(i, report.CallStart)] + teardown_wait_ends = [i for i in teardown_events if isinstance(i, report.CallFinish)] + assert len(teardown_wait_starts) == len(hosts) + assert len(teardown_wait_ends) == len(hosts) From hpk at codespeak.net Wed Aug 23 16:41:54 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 16:41:54 +0200 (CEST) Subject: [py-svn] r31555 - py/branch/distributed/py/test/rsession Message-ID: <20060823144154.F21FE10050@code0.codespeak.net> Author: hpk Date: Wed Aug 23 16:41:52 2006 New Revision: 31555 Modified: py/branch/distributed/py/test/rsession/master.py Log: formatting Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 16:41:52 2006 @@ -18,7 +18,8 @@ repr_outcome = ReprOutcome(outcomestring) item = self.pending.pop() # send finish report - self.reporter(report.ReceivedItemOutcome(self.channel, item, repr_outcome)) + self.reporter(report.ReceivedItemOutcome( + self.channel, item, repr_outcome)) def send(self, item): self.pending.insert(0, item) From hpk at codespeak.net Wed Aug 23 16:59:32 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 16:59:32 +0200 (CEST) Subject: [py-svn] r31556 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823145932.1DD8310068@code0.codespeak.net> Author: hpk Date: Wed Aug 23 16:59:29 2006 New Revision: 31556 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_master.py Log: (fijal, hpk) always assume that itemspec's have a containing collector around the package Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 16:59:29 2006 @@ -23,7 +23,7 @@ def send(self, item): self.pending.insert(0, item) - itemspec = item.listnames() + itemspec = item.listnames()[1:] self.channel.send(itemspec) # send start report self.reporter(report.SendItem(self.channel, item)) Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 16:59:29 2006 @@ -67,6 +67,7 @@ nodes = init_hosts(self.report, sshhosts, relpath, rootdir) itemgenerator = colitems[0].tryiter() + #assert 0, "\n".join([",".join(x.listnames()) for x in list(itemgenerator)]) dispatch_loop(nodes, itemgenerator, lambda : False) teardown_hosts(self.report, [node.channel for node in nodes]) Modified: py/branch/distributed/py/test/rsession/testing/test_master.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_master.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Wed Aug 23 16:59:29 2006 @@ -89,9 +89,7 @@ rootcol = py.test.collect.Directory(rootdir) funcpass_item = rootcol.getitembynames(funcpass_spec) funcfail_item = rootcol.getitembynames(funcfail_spec) - # XXX: Just make this test pass - funcpass_item.parent.parent.parent.parent.parent.parent = None - funcfail_item.parent.parent.parent.parent.parent.parent = None - itemgenerator = iter([funcfail_item] + [funcpass_item] * 5 + [funcfail_item] * 5) + itemgenerator = iter([funcfail_item] + + [funcpass_item] * 5 + [funcfail_item] * 5) shouldstop = lambda : False dispatch_loop(master_nodes, itemgenerator, shouldstop) From fijal at codespeak.net Wed Aug 23 17:08:44 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 Aug 2006 17:08:44 +0200 (CEST) Subject: [py-svn] r31557 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823150844.0DC4F10060@code0.codespeak.net> Author: fijal Date: Wed Aug 23 17:08:41 2006 New Revision: 31557 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (hpk, fijal) - Added another rsession passing test which actually tests something. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 17:08:41 2006 @@ -45,7 +45,7 @@ def bin_rsync(source, sshaddress, destpath): _rsync = py.path.local.sysfind("rsync") - print "rsyncing", source, "to", sshaddress, destpath + #print "rsyncing", source, "to", sshaddress, destpath _rsync.sysexec("-az", "--delete", "--exclude=.svn", str(source), sshaddress + ":" + str(destpath)) Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Wed Aug 23 17:08:41 2006 @@ -2,20 +2,25 @@ """ Classes for representing outcomes on master and slavenode sides """ +# WARNING! is_critical is debugging flag which means something +# wrong went on a different level. Usually that means +# internal bug. + class Outcome(object): - def __init__(self, setupfailure=False, excinfo=None, skipped=False): + def __init__(self, setupfailure=False, excinfo=None, skipped=False, is_critical=False): self.passed = not excinfo and not skipped self.skipped = skipped self.setupfailure = setupfailure self.excinfo = excinfo + self.is_critical = is_critical assert bool(self.passed) + bool(excinfo) + bool(skipped) == 1 def make_repr(self): - return (self.passed, self.setupfailure, repr(self.excinfo), self.skipped) + return (self.passed, self.setupfailure, str(self.excinfo), self.skipped, self.is_critical) class ReprOutcome(object): def __init__(self, repr_tuple): - self.passed, self.setupfailure, self.excinfo, self.skipped = repr_tuple + self.passed, self.setupfailure, self.excinfo, self.skipped, self.is_critical = repr_tuple def __repr__(self): l = ["%s=%s" %(x, getattr(self, x)) Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 17:08:41 2006 @@ -22,7 +22,7 @@ ch = setup_slave(gw, relpath) nodes_lock.acquire() try: - nodes.append(MasterNode(ch, report)) + nodes.append(MasterNode(ch, reporter)) nodes_lock.notify() finally: nodes_lock.release() @@ -66,15 +66,13 @@ rootdir = getrootdir(args) nodes = init_hosts(self.report, sshhosts, relpath, rootdir) + # XXX: We're just ignoring errors from the tryiter, so skipped tests + # does not appear in output itemgenerator = colitems[0].tryiter() #assert 0, "\n".join([",".join(x.listnames()) for x in list(itemgenerator)]) dispatch_loop(nodes, itemgenerator, lambda : False) teardown_hosts(self.report, [node.channel for node in nodes]) - # XXX: Some cleanup here, like counting all the stuff we send - # and waiting for them to arrive back - # Maybe rescheduling stuff that did not landed at all - def _map2colitems(items): # first convert all path objects into collectors from py.__.test.collect import getfscollector Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Wed Aug 23 17:08:41 2006 @@ -4,6 +4,7 @@ import py from py.__.test.rsession.executor import Executor, RunExecutor +from py.__.test.rsession.outcome import Outcome class SlaveNode(object): def __init__(self, rootcollector): @@ -28,8 +29,13 @@ nextitem = receive() if nextitem is None: break - res = node.run(nextitem) - send(res) + try: + res = node.run(nextitem) + except: + excinfo = py.code.ExceptionInfo() + send(Outcome(excinfo=excinfo, is_critical=True).make_repr()) + else: + send(res) def setup(): base_path = channel.receive() # path is ready Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Wed Aug 23 17:08:41 2006 @@ -4,20 +4,22 @@ # XXX: let's skip those tests if we do not have a disthots declared -hosts = ['codespeak.net', 'snake'] +hosts = ['snake', 'snake'] import py from py.__.test.rsession import report -from py.__.test.terminal.remote import getrootdir +from py.__.test.rsession.rsession import init_hosts, teardown_hosts +from py.__.test.rsession.testing.test_slave import (funcfail_spec, + funcpass_spec, funcskip_spec) + +def setup_module(mod): + mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() def test_setup_teardown_ssh(): - from py.__.test.rsession.rsession import init_hosts, teardown_hosts setup_events = [] teardown_events = [] - rootdir = str(py.path.local(py.__file__).dirpath()) - nodes = init_hosts(setup_events.append, hosts, 'pytestest', rootdir) teardown_hosts(teardown_events.append, [node.channel for node in nodes]) @@ -31,3 +33,31 @@ teardown_wait_ends = [i for i in teardown_events if isinstance(i, report.CallFinish)] assert len(teardown_wait_starts) == len(hosts) assert len(teardown_wait_ends) == len(hosts) + +def test_setup_teardown_run_ssh(): + events = [] + + nodes = init_hosts(events.append, hosts, 'pytestest', rootdir) + + from py.__.test.rsession.testing.test_executor \ + import ItemTestPassing, ItemTestFailing, ItemTestSkipping + + rootcol = py.test.collect.Directory(rootdir) + itempass = rootcol.getitembynames(funcpass_spec) + itemfail = rootcol.getitembynames(funcfail_spec) + itemskip = rootcol.getitembynames(funcskip_spec) + + # actually run some tests + for node in nodes: + node.send(itempass) + node.send(itemfail) + node.send(itemskip) + + teardown_hosts(lambda x: None, [node.channel for node in nodes]) + + events = [i for i in events if isinstance(i, report.ReceivedItemOutcome)] + passed = [i for i in events if i.outcome.passed] + skipped = [i for i in events if i.outcome.skipped] + assert len(passed) == len(nodes) + assert len(skipped) == len(nodes) + assert len(events) == 3 * len(nodes) From hpk at codespeak.net Wed Aug 23 17:14:48 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 17:14:48 +0200 (CEST) Subject: [py-svn] r31558 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060823151448.2D2391006C@code0.codespeak.net> Author: hpk Date: Wed Aug 23 17:14:45 2006 New Revision: 31558 Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (fijal, hpk) only run distribution tests if -D is specified Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Wed Aug 23 17:14:45 2006 @@ -2,10 +2,6 @@ """ Tests various aspects of rsession, like ssh hosts setup/teardown """ -# XXX: let's skip those tests if we do not have a disthots declared - -hosts = ['snake', 'snake'] - import py from py.__.test.rsession import report from py.__.test.rsession.rsession import init_hosts, teardown_hosts @@ -14,6 +10,10 @@ def setup_module(mod): mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() + from py.__.test.rsession.conftest import option + if not option.disthosts: + py.test.skip("no test distribution ssh hosts specified") + mod.hosts = option.disthosts.split(",") def test_setup_teardown_ssh(): @@ -29,8 +29,10 @@ assert len(count_rsyn_ends) == len(hosts) # same for teardown events - teardown_wait_starts = [i for i in teardown_events if isinstance(i, report.CallStart)] - teardown_wait_ends = [i for i in teardown_events if isinstance(i, report.CallFinish)] + teardown_wait_starts = [i for i in teardown_events + if isinstance(i, report.CallStart)] + teardown_wait_ends = [i for i in teardown_events + if isinstance(i, report.CallFinish)] assert len(teardown_wait_starts) == len(hosts) assert len(teardown_wait_ends) == len(hosts) From hpk at codespeak.net Wed Aug 23 17:15:06 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 17:15:06 +0200 (CEST) Subject: [py-svn] r31559 - py/branch/distributed/py/test/rsession Message-ID: <20060823151506.7FA6910069@code0.codespeak.net> Author: hpk Date: Wed Aug 23 17:15:04 2006 New Revision: 31559 Added: py/branch/distributed/py/test/rsession/conftest.py (contents, props changed) Log: missing part of last commit Added: py/branch/distributed/py/test/rsession/conftest.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/conftest.py Wed Aug 23 17:15:04 2006 @@ -0,0 +1,8 @@ +import py +Option = py.test.Config.Option + +option = py.test.Config.addoptions("distributed testing options", + Option('-D', '', + action="store", dest="disthosts", default=None, + help="comma separated list of testhosts", + )) From fijal at codespeak.net Wed Aug 23 17:55:42 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 Aug 2006 17:55:42 +0200 (CEST) Subject: [py-svn] r31561 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823155542.73FBF1006C@code0.codespeak.net> Author: fijal Date: Wed Aug 23 17:55:39 2006 New Revision: 31561 Modified: py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (hpk, fijal) - Correct the rootdir issue. Now it usually concerns the parent dir. Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 17:55:39 2006 @@ -17,7 +17,7 @@ nodes_lock = threading.Condition() def host_init(host): - report.wrapcall(reporter, bin_rsync, rootdir, host, relpath) + report.wrapcall(reporter, bin_rsync, str(rootdir)+'/', host, relpath) gw = py.execnet.SshGateway(host) ch = setup_slave(gw, relpath) nodes_lock.acquire() Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Wed Aug 23 17:55:39 2006 @@ -11,7 +11,7 @@ def setup_module(mod): mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() from py.__.test.rsession.conftest import option - if not option.disthosts: + if not option.disthosts: py.test.skip("no test distribution ssh hosts specified") mod.hosts = option.disthosts.split(",") @@ -37,9 +37,10 @@ assert len(teardown_wait_ends) == len(hosts) def test_setup_teardown_run_ssh(): - events = [] + allevents = [] + hosts = ['codespeak.net'] - nodes = init_hosts(events.append, hosts, 'pytestest', rootdir) + nodes = init_hosts(allevents.append, hosts, 'pytestest', rootdir) from py.__.test.rsession.testing.test_executor \ import ItemTestPassing, ItemTestFailing, ItemTestSkipping @@ -55,9 +56,9 @@ node.send(itemfail) node.send(itemskip) - teardown_hosts(lambda x: None, [node.channel for node in nodes]) + teardown_hosts(allevents.append, [node.channel for node in nodes]) - events = [i for i in events if isinstance(i, report.ReceivedItemOutcome)] + events = [i for i in allevents if isinstance(i, report.ReceivedItemOutcome)] passed = [i for i in events if i.outcome.passed] skipped = [i for i in events if i.outcome.skipped] assert len(passed) == len(nodes) From hpk at codespeak.net Wed Aug 23 18:16:15 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 18:16:15 +0200 (CEST) Subject: [py-svn] r31563 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823161615.AA4BB1006F@code0.codespeak.net> Author: hpk Date: Wed Aug 23 18:16:11 2006 New Revision: 31563 Added: py/branch/distributed/py/test/rsession/testing/test_outcome.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_master.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (fijal, hpk) use pkgdir and pkdir.dirpath() more consistently. Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 18:16:11 2006 @@ -12,12 +12,16 @@ from py.__.test.rsession.master import ( setup_slave, bin_rsync, MasterNode, dispatch_loop) -def init_hosts(reporter, sshhosts, relpath, rootdir): +def init_hosts(reporter, sshhosts, relpath, pkgdir): + assert pkgdir.join("__init__.py").check(), ( + "%s probably wrong" %(pkgdir,)) + assert relpath, relpath nodes = [] nodes_lock = threading.Condition() def host_init(host): - report.wrapcall(reporter, bin_rsync, str(rootdir)+'/', host, relpath) + report.wrapcall(reporter, bin_rsync, + str(pkgdir.dirpath())+"/", host, relpath) gw = py.execnet.SshGateway(host) ch = setup_slave(gw, relpath) nodes_lock.acquire() @@ -63,8 +67,8 @@ colitems = self._map2colitems(args) sshhosts = self.config.getinitialvalue("disthosts") relpath = "pytestcache" - rootdir = getrootdir(args) - nodes = init_hosts(self.report, sshhosts, relpath, rootdir) + pkgdir = getrootdir(args) + nodes = init_hosts(self.report, sshhosts, relpath, pkgdir) # XXX: We're just ignoring errors from the tryiter, so skipped tests # does not appear in output Modified: py/branch/distributed/py/test/rsession/testing/test_master.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_master.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Wed Aug 23 18:16:11 2006 @@ -11,7 +11,7 @@ from py.__.test.rsession import report def setup_module(mod): - mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() + mod.pkgdir = py.path.local(py.__file__).dirpath() class DummyChannel(object): def __init__(self): @@ -61,7 +61,7 @@ def test_slave_setup(): gw = py.execnet.PopenGateway() - channel = setup_slave(gw, rootdir) + channel = setup_slave(gw, pkgdir.dirpath()) channel.send(funcpass_spec) output = ReprOutcome(channel.receive()) assert output.passed @@ -81,12 +81,12 @@ def open_gw(): gw = py.execnet.PopenGateway() - channel = setup_slave(gw, rootdir) + channel = setup_slave(gw, pkgdir.dirpath()) mn = MasterNode(channel, simple_report) return mn master_nodes = [open_gw(), open_gw(), open_gw()] - rootcol = py.test.collect.Directory(rootdir) + rootcol = py.test.collect.Directory(pkgdir.dirpath()) funcpass_item = rootcol.getitembynames(funcpass_spec) funcfail_item = rootcol.getitembynames(funcfail_spec) itemgenerator = iter([funcfail_item] + Added: py/branch/distributed/py/test/rsession/testing/test_outcome.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_outcome.py Wed Aug 23 18:16:11 2006 @@ -0,0 +1,9 @@ + +from py.__.test.rsession.outcome import Outcome, ReprOutcome + +def test_critical_debugging_flag(): + outcome = Outcome(is_critical=True) + r = ReprOutcome(outcome.make_repr()) + assert r.is_critical + + Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Wed Aug 23 18:16:11 2006 @@ -9,23 +9,25 @@ funcpass_spec, funcskip_spec) def setup_module(mod): - mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() + mod.pkgdir = py.path.local(py.__file__).dirpath() from py.__.test.rsession.conftest import option if not option.disthosts: py.test.skip("no test distribution ssh hosts specified") mod.hosts = option.disthosts.split(",") def test_setup_teardown_ssh(): - setup_events = [] teardown_events = [] - nodes = init_hosts(setup_events.append, hosts, 'pytestest', rootdir) - teardown_hosts(teardown_events.append, [node.channel for node in nodes]) + nodes = init_hosts(setup_events.append, hosts, 'pytestest', pkgdir) + teardown_hosts(teardown_events.append, + [node.channel for node in nodes]) - count_rsyn_calls = [i for i in setup_events if isinstance(i, report.CallStart)] + count_rsyn_calls = [i for i in setup_events + if isinstance(i, report.CallStart)] assert len(count_rsyn_calls) == len(hosts) - count_rsyn_ends = [i for i in setup_events if isinstance(i, report.CallFinish)] + count_rsyn_ends = [i for i in setup_events + if isinstance(i, report.CallFinish)] assert len(count_rsyn_ends) == len(hosts) # same for teardown events @@ -38,14 +40,13 @@ def test_setup_teardown_run_ssh(): allevents = [] - hosts = ['codespeak.net'] - nodes = init_hosts(allevents.append, hosts, 'pytestest', rootdir) + nodes = init_hosts(allevents.append, hosts, 'pytestest', pkgdir) from py.__.test.rsession.testing.test_executor \ import ItemTestPassing, ItemTestFailing, ItemTestSkipping - rootcol = py.test.collect.Directory(rootdir) + rootcol = py.test.collect.Directory(pkgdir.dirpath()) itempass = rootcol.getitembynames(funcpass_spec) itemfail = rootcol.getitembynames(funcfail_spec) itemskip = rootcol.getitembynames(funcskip_spec) @@ -58,9 +59,12 @@ teardown_hosts(allevents.append, [node.channel for node in nodes]) - events = [i for i in allevents if isinstance(i, report.ReceivedItemOutcome)] - passed = [i for i in events if i.outcome.passed] - skipped = [i for i in events if i.outcome.skipped] + events = [i for i in allevents + if isinstance(i, report.ReceivedItemOutcome)] + passed = [i for i in events + if i.outcome.passed] + skipped = [i for i in events + if i.outcome.skipped] assert len(passed) == len(nodes) assert len(skipped) == len(nodes) assert len(events) == 3 * len(nodes) From hpk at codespeak.net Wed Aug 23 18:17:56 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 18:17:56 +0200 (CEST) Subject: [py-svn] r31564 - py/branch/distributed/py/test/rsession Message-ID: <20060823161756.D2B9F10060@code0.codespeak.net> Author: hpk Date: Wed Aug 23 18:17:53 2006 New Revision: 31564 Modified: py/branch/distributed/py/test/rsession/master.py Log: (hpk, fijal) cleanup + add rsync safety Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 18:17:53 2006 @@ -45,7 +45,7 @@ def bin_rsync(source, sshaddress, destpath): _rsync = py.path.local.sysfind("rsync") - #print "rsyncing", source, "to", sshaddress, destpath + assert destpath _rsync.sysexec("-az", "--delete", "--exclude=.svn", str(source), sshaddress + ":" + str(destpath)) @@ -54,22 +54,3 @@ ch = gateway.remote_exec(py.code.Source(slave.setup, "setup()")) ch.send(str(destpath)) return ch - - - -#def setup_(gateway, rootdir, report, path="pytest_cache"): -# bin_rsync(rootdir, gateway.sshaddress, path) -# return MasterNode(ch, report) -# -#def main(): - -# master side: -# gw = SshGateway(sshhost) -# -# setup_slave(gw) -# path = "pytest_cache" -# rsync(rootdir, gw.sshhost, path) -# pytest_cache -az --delete --exclude=.svn %(source)s %(sshhost)s:%(path)s -# c = gw.remote_exec(slave_main) -# c.send(path) -# From hpk at codespeak.net Wed Aug 23 18:47:58 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 18:47:58 +0200 (CEST) Subject: [py-svn] r31569 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823164758.853911007C@code0.codespeak.net> Author: hpk Date: Wed Aug 23 18:47:54 2006 New Revision: 31569 Modified: py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_master.py py/branch/distributed/py/test/rsession/testing/test_slave.py Log: (fijal, hpk) more unification towards pkgdir and adding tests and assertions about it. Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Wed Aug 23 18:47:54 2006 @@ -49,8 +49,8 @@ _rsync.sysexec("-az", "--delete", "--exclude=.svn", str(source), sshaddress + ":" + str(destpath)) -def setup_slave(gateway, destpath): +def setup_slave(gateway, pkgpath): from py.__.test.rsession import slave ch = gateway.remote_exec(py.code.Source(slave.setup, "setup()")) - ch.send(str(destpath)) + ch.send(str(pkgpath)) return ch Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 18:47:54 2006 @@ -2,6 +2,7 @@ """ Remote session base class """ +import os import py import thread import threading @@ -23,7 +24,7 @@ report.wrapcall(reporter, bin_rsync, str(pkgdir.dirpath())+"/", host, relpath) gw = py.execnet.SshGateway(host) - ch = setup_slave(gw, relpath) + ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) nodes_lock.acquire() try: nodes.append(MasterNode(ch, reporter)) Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Wed Aug 23 18:47:54 2006 @@ -38,9 +38,15 @@ send(res) def setup(): - base_path = channel.receive() # path is ready import os, sys - base_path = os.path.abspath(base_path) - sys.path.insert(0, base_path) + pkgdir = channel.receive() # path is ready + basedir = os.path.dirname(pkgdir) + pkgname = os.path.basename(pkgdir) + sys.path.insert(0, basedir) + import py + # XXX the following assumes that py lib is there, a bit + # much of an assumtion + mod = __import__(pkgname) + assert py.path.local(mod.__file__).dirpath() == py.path.local(pkgdir) from py.__.test.rsession.slave import slave_main - slave_main(channel.receive, channel.send, base_path) + slave_main(channel.receive, channel.send, basedir) Modified: py/branch/distributed/py/test/rsession/testing/test_master.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_master.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Wed Aug 23 18:47:54 2006 @@ -61,7 +61,7 @@ def test_slave_setup(): gw = py.execnet.PopenGateway() - channel = setup_slave(gw, pkgdir.dirpath()) + channel = setup_slave(gw, pkgdir) channel.send(funcpass_spec) output = ReprOutcome(channel.receive()) assert output.passed @@ -81,7 +81,7 @@ def open_gw(): gw = py.execnet.PopenGateway() - channel = setup_slave(gw, pkgdir.dirpath()) + channel = setup_slave(gw, pkgdir) mn = MasterNode(channel, simple_report) return mn Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Wed Aug 23 18:47:54 2006 @@ -89,3 +89,43 @@ def test_slave_run_different_stuff(): node = gettestnode() node.run("py documentation log.txt".split()) + +def test_slave_setup_fails_on_import_error(): + from py.__.test.rsession.slave import setup + tmp = py.test.ensuretemp("slavesetup") + class C: + def receive(self): + return str(tmp) + try: + exec py.code.Source(setup, "setup()").compile() in { + 'channel': C()} + except ImportError: + pass # expected + else: + py.test.fail("missing exception") + +def test_slave_setup_fails_on_missing_pkg(): + from py.__.test.rsession.slave import setup + tmp = py.test.ensuretemp("slavesetup2") + x = tmp.ensure("sometestpackage", "__init__.py") + class C: + def receive(self): + return str(x.dirpath()) + try: + exec py.code.Source(setup, "setup()").compile() in {'channel': C()} + except AttributeError: # channel.send + pass + else: + py.test.fail("missing exception") + + # now create a parallel structure + tmp = py.test.ensuretemp("slavesetup3") + x = tmp.ensure("sometestpackage", "__init__.py") + try: + exec py.code.Source(setup, "setup()").compile() in { + 'channel': C()} + except AssertionError: + pass # expected + else: + py.test.fail("missing exception") + From hpk at codespeak.net Wed Aug 23 18:57:01 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 18:57:01 +0200 (CEST) Subject: [py-svn] r31573 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060823165701.D0B1E1006C@code0.codespeak.net> Author: hpk Date: Wed Aug 23 18:56:59 2006 New Revision: 31573 Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: intermediate test for non existent ssh host checking Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Wed Aug 23 18:56:59 2006 @@ -15,6 +15,13 @@ py.test.skip("no test distribution ssh hosts specified") mod.hosts = option.disthosts.split(",") +def test_setup_non_existing_hosts(): + setup_events = [] + hosts = ["alskdjalsdkjasldkajlsd"] + nodes = init_hosts(setup_events.append, hosts, 'pytestest', pkgdir) + assert 0 + assert setup_events + def test_setup_teardown_ssh(): setup_events = [] teardown_events = [] From hpk at codespeak.net Wed Aug 23 19:00:50 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 19:00:50 +0200 (CEST) Subject: [py-svn] r31574 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060823170050.415451006C@code0.codespeak.net> Author: hpk Date: Wed Aug 23 19:00:47 2006 New Revision: 31574 Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: making a minimal check Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Wed Aug 23 19:00:47 2006 @@ -18,9 +18,9 @@ def test_setup_non_existing_hosts(): setup_events = [] hosts = ["alskdjalsdkjasldkajlsd"] - nodes = init_hosts(setup_events.append, hosts, 'pytestest', pkgdir) - assert 0 - assert setup_events + cmd = "init_hosts(setup_events.append, hosts, 'pytestest', pkgdir)" + py.test.raises(py.process.cmdexec.Error, cmd) + #assert setup_events def test_setup_teardown_ssh(): setup_events = [] From fijal at codespeak.net Wed Aug 23 19:01:34 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 Aug 2006 19:01:34 +0200 (CEST) Subject: [py-svn] r31575 - py/branch/distributed/py/test/rsession Message-ID: <20060823170134.9557E1006C@code0.codespeak.net> Author: fijal Date: Wed Aug 23 19:01:32 2006 New Revision: 31575 Modified: py/branch/distributed/py/test/rsession/rsession.py Log: Added exception re-raise. Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 19:01:32 2006 @@ -6,6 +6,7 @@ import py import thread import threading +import sys from py.__.test.rsession import report @@ -19,29 +20,37 @@ assert relpath, relpath nodes = [] nodes_lock = threading.Condition() + exc_info = [None] def host_init(host): - report.wrapcall(reporter, bin_rsync, - str(pkgdir.dirpath())+"/", host, relpath) - gw = py.execnet.SshGateway(host) - ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) - nodes_lock.acquire() try: - nodes.append(MasterNode(ch, reporter)) - nodes_lock.notify() - finally: - nodes_lock.release() + report.wrapcall(reporter, bin_rsync, + str(pkgdir.dirpath())+"/", host, relpath) + gw = py.execnet.SshGateway(host) + ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) + nodes_lock.acquire() + try: + nodes.append(MasterNode(ch, reporter)) + nodes_lock.notify() + finally: + nodes_lock.release() + except: + exc_info[0] = sys.exc_info() for host in sshhosts: thread.start_new_thread(host_init, (host,)) while len(nodes) != len(sshhosts): + if exc_info != [None]: + exc, data, tb = exc_info[0] + raise exc, data, tb nodes_lock.acquire() try: nodes_lock.wait(1) finally: nodes_lock.release() - + + return nodes def teardown_hosts(reporter, channels): From hpk at codespeak.net Wed Aug 23 19:09:20 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 19:09:20 +0200 (CEST) Subject: [py-svn] r31577 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060823170920.2EDA710069@code0.codespeak.net> Author: hpk Date: Wed Aug 23 19:09:17 2006 New Revision: 31577 Added: py/branch/distributed/py/test/rsession/hostmanage.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (fijal, hpk) reshuffle init_hosts/teardown stuff into hostmanage.py Added: py/branch/distributed/py/test/rsession/hostmanage.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/hostmanage.py Wed Aug 23 19:09:17 2006 @@ -0,0 +1,54 @@ +import sys, os +import py +import thread, threading +from py.__.test.rsession.master import ( + setup_slave, bin_rsync, MasterNode, dispatch_loop) +from py.__.test.rsession import report + +def init_hosts(reporter, sshhosts, relpath, pkgdir): + assert pkgdir.join("__init__.py").check(), ( + "%s probably wrong" %(pkgdir,)) + assert relpath, relpath + nodes = [] + nodes_lock = threading.Condition() + exc_info = [None] + + def host_init(host): + try: + report.wrapcall(reporter, bin_rsync, + str(pkgdir.dirpath())+"/", host, relpath) + gw = py.execnet.SshGateway(host) + ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) + nodes_lock.acquire() + try: + nodes.append(MasterNode(ch, reporter)) + nodes_lock.notify() + finally: + nodes_lock.release() + except: + exc_info[0] = sys.exc_info() + + for host in sshhosts: + thread.start_new_thread(host_init, (host,)) + + while len(nodes) != len(sshhosts): + if exc_info != [None]: + exc, data, tb = exc_info[0] + raise exc, data, tb + nodes_lock.acquire() + try: + nodes_lock.wait(1) + finally: + nodes_lock.release() + + + return nodes + +def teardown_hosts(reporter, channels): + for channel in channels: + channel.send(None) + + for channel in channels: + report.wrapcall(reporter, channel.waitclose) + channel.gateway.exit() + Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Wed Aug 23 19:09:17 2006 @@ -4,62 +4,13 @@ import os import py -import thread -import threading import sys from py.__.test.rsession import report from py.__.test.terminal.remote import getrootdir -from py.__.test.rsession.master import ( - setup_slave, bin_rsync, MasterNode, dispatch_loop) - -def init_hosts(reporter, sshhosts, relpath, pkgdir): - assert pkgdir.join("__init__.py").check(), ( - "%s probably wrong" %(pkgdir,)) - assert relpath, relpath - nodes = [] - nodes_lock = threading.Condition() - exc_info = [None] - - def host_init(host): - try: - report.wrapcall(reporter, bin_rsync, - str(pkgdir.dirpath())+"/", host, relpath) - gw = py.execnet.SshGateway(host) - ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) - nodes_lock.acquire() - try: - nodes.append(MasterNode(ch, reporter)) - nodes_lock.notify() - finally: - nodes_lock.release() - except: - exc_info[0] = sys.exc_info() - - for host in sshhosts: - thread.start_new_thread(host_init, (host,)) - - while len(nodes) != len(sshhosts): - if exc_info != [None]: - exc, data, tb = exc_info[0] - raise exc, data, tb - nodes_lock.acquire() - try: - nodes_lock.wait(1) - finally: - nodes_lock.release() - - - return nodes - -def teardown_hosts(reporter, channels): - for channel in channels: - channel.send(None) - - for channel in channels: - report.wrapcall(reporter, channel.waitclose) - channel.gateway.exit() +from py.__.test.rsession.master import dispatch_loop +from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts class RSession(object): """ Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Wed Aug 23 19:09:17 2006 @@ -4,7 +4,7 @@ import py from py.__.test.rsession import report -from py.__.test.rsession.rsession import init_hosts, teardown_hosts +from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts from py.__.test.rsession.testing.test_slave import (funcfail_spec, funcpass_spec, funcskip_spec) From hpk at codespeak.net Thu Aug 24 12:07:23 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 12:07:23 +0200 (CEST) Subject: [py-svn] r31587 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060824100723.B55B410036@code0.codespeak.net> Author: hpk Date: Thu Aug 24 12:07:21 2006 New Revision: 31587 Modified: py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (fijal, hpk) cleanup initialisation and transformation from filesystem paths to collectors. Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Thu Aug 24 12:07:21 2006 @@ -7,8 +7,6 @@ import sys from py.__.test.rsession import report - -from py.__.test.terminal.remote import getrootdir from py.__.test.rsession.master import dispatch_loop from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts @@ -38,16 +36,26 @@ dispatch_loop(nodes, itemgenerator, lambda : False) teardown_hosts(self.report, [node.channel for node in nodes]) - def _map2colitems(items): - # first convert all path objects into collectors - from py.__.test.collect import getfscollector - colitems = [] - for item in items: - if isinstance(item, (list, tuple)): - colitems.extend(Session._map2colitems(item)) - elif not isinstance(item, py.test.collect.Collector): - colitems.append(getfscollector(item)) - else: - colitems.append(item) - return colitems - _map2colitems = staticmethod(_map2colitems) + def make_colitems(paths, baseon): + # we presume that from the base we can simply get to + # the target paths by joining the basenames + res = [] + for x in paths: + current = py.test.collect.Directory(baseon) + relparts = x.relto(baseon).split(x.sep) + assert relparts + for part in relparts: + next = current.join(part) + assert next is not None, (current, part) + current = next + res.append(current) + return res + make_colitems = staticmethod(make_colitems) + + def getpkgdir(path): + path = py.path.local(path) + pkgpath = path.pypkgpath() + assert pkgpath is not None, ( + "no pkgdir for global scoped file %s" %(path,)) + return pkgpath + getpkgdir = staticmethod(getpkgdir) Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Thu Aug 24 12:07:21 2006 @@ -4,16 +4,13 @@ import py from py.__.test.rsession import report +from py.__.test.rsession.rsession import RSession from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts from py.__.test.rsession.testing.test_slave import (funcfail_spec, funcpass_spec, funcskip_spec) def setup_module(mod): mod.pkgdir = py.path.local(py.__file__).dirpath() - from py.__.test.rsession.conftest import option - if not option.disthosts: - py.test.skip("no test distribution ssh hosts specified") - mod.hosts = option.disthosts.split(",") def test_setup_non_existing_hosts(): setup_events = [] @@ -21,57 +18,97 @@ cmd = "init_hosts(setup_events.append, hosts, 'pytestest', pkgdir)" py.test.raises(py.process.cmdexec.Error, cmd) #assert setup_events - -def test_setup_teardown_ssh(): - setup_events = [] - teardown_events = [] - - nodes = init_hosts(setup_events.append, hosts, 'pytestest', pkgdir) - teardown_hosts(teardown_events.append, - [node.channel for node in nodes]) - - count_rsyn_calls = [i for i in setup_events - if isinstance(i, report.CallStart)] - assert len(count_rsyn_calls) == len(hosts) - count_rsyn_ends = [i for i in setup_events - if isinstance(i, report.CallFinish)] - assert len(count_rsyn_ends) == len(hosts) - - # same for teardown events - teardown_wait_starts = [i for i in teardown_events - if isinstance(i, report.CallStart)] - teardown_wait_ends = [i for i in teardown_events - if isinstance(i, report.CallFinish)] - assert len(teardown_wait_starts) == len(hosts) - assert len(teardown_wait_ends) == len(hosts) - -def test_setup_teardown_run_ssh(): - allevents = [] - - nodes = init_hosts(allevents.append, hosts, 'pytestest', pkgdir) - - from py.__.test.rsession.testing.test_executor \ - import ItemTestPassing, ItemTestFailing, ItemTestSkipping - - rootcol = py.test.collect.Directory(pkgdir.dirpath()) - itempass = rootcol.getitembynames(funcpass_spec) - itemfail = rootcol.getitembynames(funcfail_spec) - itemskip = rootcol.getitembynames(funcskip_spec) - - # actually run some tests - for node in nodes: - node.send(itempass) - node.send(itemfail) - node.send(itemskip) - - teardown_hosts(allevents.append, [node.channel for node in nodes]) - - events = [i for i in allevents - if isinstance(i, report.ReceivedItemOutcome)] - passed = [i for i in events - if i.outcome.passed] - skipped = [i for i in events - if i.outcome.skipped] - assert len(passed) == len(nodes) - assert len(skipped) == len(nodes) - assert len(events) == 3 * len(nodes) + +def test_getpkdir(): + one = pkgdir.join("initpkg.py") + two = pkgdir.join("path", "__init__.py") + p1 = RSession.getpkgdir(one) + p2 = RSession.getpkgdir(two) + assert p1 == p2 + assert p1 == pkgdir + +def test_getpkdir_no_inits(): + tmp = py.test.ensuretemp("getpkdir1") + fn = tmp.ensure("hello.py") + py.test.raises(AssertionError, "RSession.getpkgdir(fn)") + +def test_make_colitems(): + one = pkgdir.join("initpkg.py") + two = pkgdir.join("path", "__init__.py") + + cols = RSession.make_colitems([one, two], baseon=pkgdir) + assert len(cols) == 2 + col_one, col_two = cols + assert col_one.listnames() == ["py", "initpkg.py"] + assert col_two.listnames() == ["py", "path", "__init__.py"] + + cols = RSession.make_colitems([one, two], baseon=pkgdir.dirpath()) + assert len(cols) == 2 + col_one, col_two = cols + assert col_one.listnames() == [pkgdir.dirpath().basename, + "py", "initpkg.py"] + assert col_two.listnames() == [pkgdir.dirpath().basename, + "py", "path", "__init__.py"] + +class TestWithRealSshHosts: + def setup_class(cls): + from py.__.test.rsession.conftest import option + if not option.disthosts: + py.test.skip("no test distribution ssh hosts specified") + cls.hosts = option.disthosts.split(",") + + def test_setup_teardown_ssh(self): + hosts = self.hosts + setup_events = [] + teardown_events = [] + + nodes = init_hosts(setup_events.append, hosts, 'pytestest', pkgdir) + teardown_hosts(teardown_events.append, + [node.channel for node in nodes]) + + count_rsyn_calls = [i for i in setup_events + if isinstance(i, report.CallStart)] + assert len(count_rsyn_calls) == len(hosts) + count_rsyn_ends = [i for i in setup_events + if isinstance(i, report.CallFinish)] + assert len(count_rsyn_ends) == len(hosts) + + # same for teardown events + teardown_wait_starts = [i for i in teardown_events + if isinstance(i, report.CallStart)] + teardown_wait_ends = [i for i in teardown_events + if isinstance(i, report.CallFinish)] + assert len(teardown_wait_starts) == len(hosts) + assert len(teardown_wait_ends) == len(hosts) + + def test_setup_teardown_run_ssh(self): + hosts = self.hosts + allevents = [] + + nodes = init_hosts(allevents.append, hosts, 'pytestest', pkgdir) + + from py.__.test.rsession.testing.test_executor \ + import ItemTestPassing, ItemTestFailing, ItemTestSkipping + + rootcol = py.test.collect.Directory(pkgdir.dirpath()) + itempass = rootcol.getitembynames(funcpass_spec) + itemfail = rootcol.getitembynames(funcfail_spec) + itemskip = rootcol.getitembynames(funcskip_spec) + + # actually run some tests + for node in nodes: + node.send(itempass) + node.send(itemfail) + node.send(itemskip) + + teardown_hosts(allevents.append, [node.channel for node in nodes]) + + events = [i for i in allevents + if isinstance(i, report.ReceivedItemOutcome)] + passed = [i for i in events + if i.outcome.passed] + skipped = [i for i in events + if i.outcome.skipped] + assert len(passed) == len(nodes) + assert len(skipped) == len(nodes) + assert len(events) == 3 * len(nodes) From hpk at codespeak.net Thu Aug 24 12:20:07 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 12:20:07 +0200 (CEST) Subject: [py-svn] r31588 - py/branch/distributed/py/test/rsession Message-ID: <20060824102007.BD4A21005A@code0.codespeak.net> Author: hpk Date: Thu Aug 24 12:20:05 2006 New Revision: 31588 Modified: py/branch/distributed/py/test/rsession/rsession.py Log: (fijal, hpk) make the RSession basically work Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Thu Aug 24 12:20:05 2006 @@ -23,15 +23,21 @@ def main(self, args): """ main loop for running tests. """ - colitems = self._map2colitems(args) + if not args: + args = [py.path.local()] + pkgdir = self.getpkgdir(args[0]) + colitems = self.make_colitems(args, baseon=pkgdir.dirpath()) + sshhosts = self.config.getinitialvalue("disthosts") - relpath = "pytestcache" - pkgdir = getrootdir(args) - nodes = init_hosts(self.report, sshhosts, relpath, pkgdir) + destrelpath = "pytestcache" + nodes = init_hosts(self.report, sshhosts, destrelpath, pkgdir) - # XXX: We're just ignoring errors from the tryiter, so skipped tests - # does not appear in output - itemgenerator = colitems[0].tryiter() + def itemgen(): + for x in colitems: + for y in x.tryiter(): + yield y + + itemgenerator = itemgen() #assert 0, "\n".join([",".join(x.listnames()) for x in list(itemgenerator)]) dispatch_loop(nodes, itemgenerator, lambda : False) teardown_hosts(self.report, [node.channel for node in nodes]) @@ -41,6 +47,7 @@ # the target paths by joining the basenames res = [] for x in paths: + x = py.path.local(x) current = py.test.collect.Directory(baseon) relparts = x.relto(baseon).split(x.sep) assert relparts From fijal at codespeak.net Thu Aug 24 12:22:15 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 24 Aug 2006 12:22:15 +0200 (CEST) Subject: [py-svn] r31589 - py/branch/distributed/py/test/rsession Message-ID: <20060824102215.E96541005A@code0.codespeak.net> Author: fijal Date: Thu Aug 24 12:22:14 2006 New Revision: 31589 Modified: py/branch/distributed/py/test/rsession/outcome.py Log: Lines split. Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Thu Aug 24 12:22:14 2006 @@ -7,7 +7,8 @@ # internal bug. class Outcome(object): - def __init__(self, setupfailure=False, excinfo=None, skipped=False, is_critical=False): + def __init__(self, setupfailure=False, excinfo=None, skipped=False, + is_critical=False): self.passed = not excinfo and not skipped self.skipped = skipped self.setupfailure = setupfailure @@ -16,11 +17,13 @@ assert bool(self.passed) + bool(excinfo) + bool(skipped) == 1 def make_repr(self): - return (self.passed, self.setupfailure, str(self.excinfo), self.skipped, self.is_critical) + return (self.passed, self.setupfailure, str(self.excinfo), + self.skipped, self.is_critical) class ReprOutcome(object): def __init__(self, repr_tuple): - self.passed, self.setupfailure, self.excinfo, self.skipped, self.is_critical = repr_tuple + (self.passed, self.setupfailure, self.excinfo, self.skipped, + self.is_critical) = repr_tuple def __repr__(self): l = ["%s=%s" %(x, getattr(self, x)) From hpk at codespeak.net Thu Aug 24 13:02:49 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 13:02:49 +0200 (CEST) Subject: [py-svn] r31592 - in py/branch/distributed/py/code: . testing Message-ID: <20060824110249.B33821005A@code0.codespeak.net> Author: hpk Date: Thu Aug 24 13:02:47 2006 New Revision: 31592 Modified: py/branch/distributed/py/code/testing/test_excinfo.py py/branch/distributed/py/code/traceback2.py Log: (fijal, hpk) safer traceback entry __str__. Modified: py/branch/distributed/py/code/testing/test_excinfo.py ============================================================================== --- py/branch/distributed/py/code/testing/test_excinfo.py (original) +++ py/branch/distributed/py/code/testing/test_excinfo.py Thu Aug 24 13:02:47 2006 @@ -182,3 +182,10 @@ def test_excinfo_errisinstance(): excinfo = py.test.raises(ValueError, h) assert excinfo.errisinstance(ValueError) + +def test_excinfo_no_sourcecode(): + try: + exec "raise ValueError()" + except ValueError: + excinfo = py.code.ExceptionInfo() + s = str(excinfo.traceback[-1]) Modified: py/branch/distributed/py/code/traceback2.py ============================================================================== --- py/branch/distributed/py/code/traceback2.py (original) +++ py/branch/distributed/py/code/traceback2.py Thu Aug 24 13:02:47 2006 @@ -66,7 +66,10 @@ except py.error.Error: fn = '???' name = self.frame.code.name - line = str(self.statement).lstrip() + try: + line = str(self.statement).lstrip() + except EnvironmentError, e: + line = "" return " File %r:%d in %s\n %s\n" %(fn, self.lineno+1, name, line) class Traceback(list): From fijal at codespeak.net Thu Aug 24 13:05:11 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 24 Aug 2006 13:05:11 +0200 (CEST) Subject: [py-svn] r31593 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060824110511.788E01005A@code0.codespeak.net> Author: fijal Date: Thu Aug 24 13:05:09 2006 New Revision: 31593 Modified: py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/testing/test_master.py py/branch/distributed/py/test/rsession/testing/test_outcome.py Log: (hpk, fijal) - Added simple traceback representation for remote tracebacks. Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Thu Aug 24 13:05:09 2006 @@ -16,14 +16,36 @@ self.is_critical = is_critical assert bool(self.passed) + bool(excinfo) + bool(skipped) == 1 + def make_excinfo_repr(self): + if self.excinfo is None: + return None + excinfo = self.excinfo + tb_info = [str(x) for x in excinfo.traceback] + return (excinfo.type.__name__, str(excinfo.value), tb_info) + def make_repr(self): - return (self.passed, self.setupfailure, str(self.excinfo), + return (self.passed, self.setupfailure, + self.make_excinfo_repr(), self.skipped, self.is_critical) +class ExcInfoRepr(object): + def __init__(self, excinfo): + self.typename, self.value, self.traceback = excinfo + + def __repr__(self): + l = ["%s=%s" %(x, getattr(self, x)) + for x in "typename value traceback".split()] + return "" %(" ".join(l),) + + class ReprOutcome(object): def __init__(self, repr_tuple): - (self.passed, self.setupfailure, self.excinfo, self.skipped, + (self.passed, self.setupfailure, excinfo, self.skipped, self.is_critical) = repr_tuple + if excinfo is None: + self.excinfo = None + else: + self.excinfo = ExcInfoRepr(excinfo) def __repr__(self): l = ["%s=%s" %(x, getattr(self, x)) Modified: py/branch/distributed/py/test/rsession/testing/test_master.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_master.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_master.py Thu Aug 24 13:05:09 2006 @@ -25,13 +25,18 @@ self.sent.append(item) def test_masternode(): + try: + raise ValueError() + except ValueError: + excinfo = py.code.ExceptionInfo() + ch = DummyChannel() reportlist = [] mnode = MasterNode(ch, reportlist.append) mnode.send(py.test.Item("ok")) mnode.send(py.test.Item("notok")) ch.callback(Outcome().make_repr()) - ch.callback(Outcome(excinfo=42).make_repr()) + ch.callback(Outcome(excinfo=excinfo).make_repr()) assert len(reportlist) == 4 received = [i for i in reportlist if isinstance(i, report.ReceivedItemOutcome)] Modified: py/branch/distributed/py/test/rsession/testing/test_outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_outcome.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_outcome.py Thu Aug 24 13:05:09 2006 @@ -1,9 +1,33 @@ -from py.__.test.rsession.outcome import Outcome, ReprOutcome +import py +from py.__.test.rsession.outcome import Outcome, ReprOutcome, ExcInfoRepr + +import marshal def test_critical_debugging_flag(): outcome = Outcome(is_critical=True) r = ReprOutcome(outcome.make_repr()) assert r.is_critical +def f1(): + raise ValueError(42) + +def f2(): + f1() + +def f3(): + f2() + +def test_exception_info_repr(): + try: + f3() + except: + outcome = Outcome(excinfo=py.code.ExceptionInfo()) + + repr = outcome.make_excinfo_repr() + assert marshal.dumps(repr) + excinfo = ExcInfoRepr(repr) + assert str(excinfo.typename) == "ValueError" + assert str(excinfo.value) == "42" + assert len(excinfo.traceback) == 4 From hpk at codespeak.net Thu Aug 24 13:20:03 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 13:20:03 +0200 (CEST) Subject: [py-svn] r31596 - py/branch/distributed/py/code Message-ID: <20060824112003.5323E10068@code0.codespeak.net> Author: hpk Date: Thu Aug 24 13:20:00 2006 New Revision: 31596 Modified: py/branch/distributed/py/code/traceback2.py Log: fix msg Modified: py/branch/distributed/py/code/traceback2.py ============================================================================== --- py/branch/distributed/py/code/traceback2.py (original) +++ py/branch/distributed/py/code/traceback2.py Thu Aug 24 13:20:00 2006 @@ -69,7 +69,7 @@ try: line = str(self.statement).lstrip() except EnvironmentError, e: - line = "" + line = "" return " File %r:%d in %s\n %s\n" %(fn, self.lineno+1, name, line) class Traceback(list): From fijal at codespeak.net Thu Aug 24 13:47:16 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 24 Aug 2006 13:47:16 +0200 (CEST) Subject: [py-svn] r31597 - py/branch/distributed/py/test/rsession Message-ID: <20060824114716.1373410068@code0.codespeak.net> Author: fijal Date: Thu Aug 24 13:47:12 2006 New Revision: 31597 Modified: py/branch/distributed/py/test/rsession/hostmanage.py py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/rsession.py Log: (hpk, fijal) - Added slightly better reporting stuff Modified: py/branch/distributed/py/test/rsession/hostmanage.py ============================================================================== --- py/branch/distributed/py/test/rsession/hostmanage.py (original) +++ py/branch/distributed/py/test/rsession/hostmanage.py Thu Aug 24 13:47:12 2006 @@ -9,6 +9,7 @@ assert pkgdir.join("__init__.py").check(), ( "%s probably wrong" %(pkgdir,)) assert relpath, relpath + assert not os.path.isabs(relpath), relpath nodes = [] nodes_lock = threading.Condition() exc_info = [None] Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Thu Aug 24 13:47:12 2006 @@ -6,6 +6,8 @@ # wrong went on a different level. Usually that means # internal bug. +import sys + class Outcome(object): def __init__(self, setupfailure=False, excinfo=None, skipped=False, is_critical=False): @@ -37,7 +39,6 @@ for x in "typename value traceback".split()] return "" %(" ".join(l),) - class ReprOutcome(object): def __init__(self, repr_tuple): (self.passed, self.setupfailure, excinfo, self.skipped, Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Thu Aug 24 13:47:12 2006 @@ -4,23 +4,54 @@ import os import py +import thread +import threading import sys from py.__.test.rsession import report -from py.__.test.rsession.master import dispatch_loop +from py.__.test.rsession.master import ( + setup_slave, bin_rsync, MasterNode, dispatch_loop) from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts +class Reporter(object): + def report(self, what): + repfun = getattr(self, "report_" + what.__class__.__name__, + self.report_unknown) + try: + repfun(what) + except (KeyboardInterrupt, SystemExit): + raise + except: + print "Internal reporting problem" + excinfo = py.code.ExceptionInfo() + print excinfo + + def report_unknown(self, what): + print "Unknown report: %s" % what + + def report_SendItem(self, item): + print "Sending %s to %s" % (item.item, item.channel.gateway.sshaddress) + + def report_ReceivedItemOutcome(self, event): + sshhost = event.channel.gateway.sshaddress + if event.outcome.passed: + print "%s passed:%s" % (event.item, sshhost) + elif event.outcome.skipped: + print "%s skipped:%s" % (event.item, sshhost) + else: + print "%s FAILED: %s" % (event.item, sshhost) + excinfo = event.outcome.excinfo + print "".join(excinfo.traceback) + print "%s: %s" % (excinfo.typename, excinfo.value) + class RSession(object): """ An RSession executes collectors/items through a runner. """ - def __init__(self, config): - self.config = config + def __init__(self, config): + self.config = config - def report(self, *whatever): - print whatever - def main(self, args): """ main loop for running tests. """ if not args: @@ -30,7 +61,8 @@ sshhosts = self.config.getinitialvalue("disthosts") destrelpath = "pytestcache" - nodes = init_hosts(self.report, sshhosts, destrelpath, pkgdir) + self.reporter = Reporter() + nodes = init_hosts(self.reporter.report, sshhosts, destrelpath, pkgdir) def itemgen(): for x in colitems: @@ -38,9 +70,10 @@ yield y itemgenerator = itemgen() - #assert 0, "\n".join([",".join(x.listnames()) for x in list(itemgenerator)]) + #assert 0, "\n".join([",".join(x.listnames()) for x in + # list(itemgenerator)]) dispatch_loop(nodes, itemgenerator, lambda : False) - teardown_hosts(self.report, [node.channel for node in nodes]) + teardown_hosts(self.reporter.report, [node.channel for node in nodes]) def make_colitems(paths, baseon): # we presume that from the base we can simply get to From hpk at codespeak.net Thu Aug 24 14:02:51 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 14:02:51 +0200 (CEST) Subject: [py-svn] r31599 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060824120251.053DB10068@code0.codespeak.net> Author: hpk Date: Thu Aug 24 14:02:47 2006 New Revision: 31599 Modified: py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_report.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (fijal, hpk) have a first simple test for the main entry point RSession.main() Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Thu Aug 24 14:02:47 2006 @@ -52,7 +52,7 @@ def __init__(self, config): self.config = config - def main(self, args): + def main(self, args, reporter=None): """ main loop for running tests. """ if not args: args = [py.path.local()] @@ -61,8 +61,9 @@ sshhosts = self.config.getinitialvalue("disthosts") destrelpath = "pytestcache" - self.reporter = Reporter() - nodes = init_hosts(self.reporter.report, sshhosts, destrelpath, pkgdir) + if reporter is None: + reporter = Reporter().report + nodes = init_hosts(reporter, sshhosts, destrelpath, pkgdir) def itemgen(): for x in colitems: @@ -73,7 +74,7 @@ #assert 0, "\n".join([",".join(x.listnames()) for x in # list(itemgenerator)]) dispatch_loop(nodes, itemgenerator, lambda : False) - teardown_hosts(self.reporter.report, [node.channel for node in nodes]) + teardown_hosts(reporter, [node.channel for node in nodes]) def make_colitems(paths, baseon): # we presume that from the base we can simply get to Modified: py/branch/distributed/py/test/rsession/testing/test_report.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_report.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_report.py Thu Aug 24 14:02:47 2006 @@ -23,4 +23,3 @@ assert len(l) == 2 assert isinstance(l[0], report.CallStart) assert isinstance(l[1], report.CallException) - Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Thu Aug 24 14:02:47 2006 @@ -57,6 +57,29 @@ py.test.skip("no test distribution ssh hosts specified") cls.hosts = option.disthosts.split(",") + def test_example_distribution(self): + # XXX find a better way for the below + tmpdir = py.path.local(py.__file__).dirpath().dirpath() + tmpdir.ensure("sub", "conftest.py").write(py.code.Source(""" + disthosts = %r + """ % self.hosts)) + tmpdir.ensure("sub", "__init__.py") + tmpdir.ensure("sub", "test_one.py").write(py.code.Source(""" + def test_1(self): + pass + def test_2(self): + assert 0 + """)) + args = [str(tmpdir.join("sub"))] + config, args = py.test.Config.parse(args) + rsession = RSession(config) + allevents = [] + rsession.main(args, reporter=allevents.append) + testevents = [x for x in allevents + if isinstance(x, report.ReceivedItemOutcome)] + assert len(testevents) + assert 0, allevents + def test_setup_teardown_ssh(self): hosts = self.hosts setup_events = [] From fijal at codespeak.net Thu Aug 24 14:17:06 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 24 Aug 2006 14:17:06 +0200 (CEST) Subject: [py-svn] r31600 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060824121706.5494F10068@code0.codespeak.net> Author: fijal Date: Thu Aug 24 14:17:04 2006 New Revision: 31600 Modified: py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/testing/test_outcome.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (hpk, fijal) - Added traceback representation for remote running. Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Thu Aug 24 14:17:04 2006 @@ -22,17 +22,33 @@ if self.excinfo is None: return None excinfo = self.excinfo - tb_info = [str(x) for x in excinfo.traceback] + tb_info = [self.traceback_entry_repr(x) for x in excinfo.traceback] return (excinfo.type.__name__, str(excinfo.value), tb_info) + def traceback_entry_repr(self, tb_entry): + try: + return (str(tb_entry.lineno), str(tb_entry.getsource()), + str(tb_entry.path)) + except: + return (0, "", "??") + def make_repr(self): return (self.passed, self.setupfailure, self.make_excinfo_repr(), self.skipped, self.is_critical) +class TracebackEntryRepr(object): + def __init__(self, tbentry): + lineno, self.source, self.path = tbentry + self.lineno = int(lineno) + + def __repr__(self): + return "line %s in %s\n %s" %(self.lineno, self.path, self.source) + class ExcInfoRepr(object): def __init__(self, excinfo): - self.typename, self.value, self.traceback = excinfo + self.typename, self.value, tb = excinfo + self.traceback = [TracebackEntryRepr(x) for x in tb] def __repr__(self): l = ["%s=%s" %(x, getattr(self, x)) Modified: py/branch/distributed/py/test/rsession/testing/test_outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_outcome.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_outcome.py Thu Aug 24 14:17:04 2006 @@ -31,3 +31,6 @@ assert str(excinfo.typename) == "ValueError" assert str(excinfo.value) == "42" assert len(excinfo.traceback) == 4 + assert excinfo.traceback[3].lineno == f1.func_code.co_firstlineno + assert excinfo.traceback[2].lineno == f2.func_code.co_firstlineno + assert excinfo.traceback[1].lineno == f3.func_code.co_firstlineno Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Thu Aug 24 14:17:04 2006 @@ -77,7 +77,7 @@ rsession.main(args, reporter=allevents.append) testevents = [x for x in allevents if isinstance(x, report.ReceivedItemOutcome)] - assert len(testevents) + assert len(testevents) assert 0, allevents def test_setup_teardown_ssh(self): From hpk at codespeak.net Thu Aug 24 14:24:54 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 14:24:54 +0200 (CEST) Subject: [py-svn] r31601 - py/branch/distributed/py/test/rsession/testing Message-ID: <20060824122454.940D010068@code0.codespeak.net> Author: hpk Date: Thu Aug 24 14:24:52 2006 New Revision: 31601 Modified: py/branch/distributed/py/test/rsession/testing/test_outcome.py Log: (fijal, hpk) adding tests for tracebackentry.path Modified: py/branch/distributed/py/test/rsession/testing/test_outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_outcome.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_outcome.py Thu Aug 24 14:24:52 2006 @@ -31,6 +31,10 @@ assert str(excinfo.typename) == "ValueError" assert str(excinfo.value) == "42" assert len(excinfo.traceback) == 4 + myfile = py.magic.autopath() + assert excinfo.traceback[3].path == myfile assert excinfo.traceback[3].lineno == f1.func_code.co_firstlineno + assert excinfo.traceback[2].path == myfile assert excinfo.traceback[2].lineno == f2.func_code.co_firstlineno + assert excinfo.traceback[1].path == myfile assert excinfo.traceback[1].lineno == f3.func_code.co_firstlineno From fijal at codespeak.net Thu Aug 24 14:44:14 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 24 Aug 2006 14:44:14 +0200 (CEST) Subject: [py-svn] r31602 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060824124414.B06B310069@code0.codespeak.net> Author: fijal Date: Thu Aug 24 14:44:08 2006 New Revision: 31602 Modified: py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (hpk, fijal) - Added a (failing) test for rsession Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Thu Aug 24 14:44:08 2006 @@ -26,12 +26,14 @@ return (excinfo.type.__name__, str(excinfo.value), tb_info) def traceback_entry_repr(self, tb_entry): + lineno = tb_entry.lineno + path = str(tb_entry.path) try: - return (str(tb_entry.lineno), str(tb_entry.getsource()), - str(tb_entry.path)) + source = str(tb_entry.getsource()) except: - return (0, "", "??") - + source = "" + return (lineno, source, path) + def make_repr(self): return (self.passed, self.setupfailure, self.make_excinfo_repr(), Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Thu Aug 24 14:44:08 2006 @@ -62,13 +62,17 @@ tmpdir = py.path.local(py.__file__).dirpath().dirpath() tmpdir.ensure("sub", "conftest.py").write(py.code.Source(""" disthosts = %r - """ % self.hosts)) + """ % self.hosts[:1])) tmpdir.ensure("sub", "__init__.py") tmpdir.ensure("sub", "test_one.py").write(py.code.Source(""" - def test_1(self): + def test_1(): pass - def test_2(self): + def test_2(): assert 0 + def test_3(): + raise ValueError(23) + def test_4(someargs): + pass """)) args = [str(tmpdir.join("sub"))] config, args = py.test.Config.parse(args) @@ -78,8 +82,25 @@ testevents = [x for x in allevents if isinstance(x, report.ReceivedItemOutcome)] assert len(testevents) - assert 0, allevents - + passevents = [i for i in testevents if i.outcome.passed] + failevents = [i for i in testevents if i.outcome.excinfo] + skippedevents = [i for i in testevents if i.outcome.skipped] + assert len(passevents) == 1 + assert len(failevents) == 3 + tb = failevents[0].outcome.excinfo.traceback + assert tb[0].path.find("test_one") != -1 + assert tb[0].source.find("test_2") != -1 + assert failevents[0].outcome.excinfo.typename == 'AssertionError' + tb = failevents[1].outcome.excinfo.traceback + assert tb[0].path.find("test_one") != -1 + assert tb[0].source.find("test_3") != -1 + assert failevents[1].outcome.excinfo.typename == 'ValueError' + assert failevents[1].outcome.excinfo.value == '23' + tb = failevents[2].outcome.excinfo.traceback + assert tb[0].path.find("test_one") != -1 + assert tb[0].source.find("test_4") != -1 + assert failevents[2].outcome.excinfo.typename == 'TypeError' + def test_setup_teardown_ssh(self): hosts = self.hosts setup_events = [] From hpk at codespeak.net Thu Aug 24 16:05:17 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 16:05:17 +0200 (CEST) Subject: [py-svn] r31608 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060824140517.81DB710060@code0.codespeak.net> Author: hpk Date: Thu Aug 24 16:05:13 2006 New Revision: 31608 Modified: py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/hostmanage.py py/branch/distributed/py/test/rsession/master.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_executor.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: (fijal, hpk) make another important test work cleanup and test of rsyncing (it used to transfer pyc files, argh) Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Thu Aug 24 16:05:13 2006 @@ -5,7 +5,31 @@ from py.__.test.rsession.outcome import Outcome -class Executor(object): +class RunExecutor(object): + """ Same as in executor, but just running run + """ + def __init__(self, item): + self.item = item + + def execute(self): + try: + self.item.run() + return Outcome() + except py.test.Item.Skipped: + return Outcome(skipped=True) + except: + excinfo = py.code.ExceptionInfo() + if isinstance(self.item, py.test.Function): + fun = self.item.obj # hope this is stable + code = py.code.Code(fun) + excinfo.traceback = excinfo.traceback.cut( + path=code.path, firstlineno=code.firstlineno) + return Outcome(excinfo=excinfo, setupfailure=False) + + + +# XXX old code +class XXXExecutor(object): def __init__(self, fun, setup=lambda: None): self.fun = fun self.setup = setup @@ -29,23 +53,3 @@ return Outcome(excinfo=excinfo, setupfailure=False) return Outcome() -class RunExecutor(Executor): - """ Same as in executor, but just running run - """ - def __init__(self, item): - self.item = item - - def execute(self): - try: - self.item.run() - return Outcome() - except py.test.Item.Skipped: - return Outcome(skipped=True) - except: - excinfo = py.code.ExceptionInfo() - if isinstance(self.item, py.test.Function): - fun = self.item.obj # hope this is stable - code = py.code.Code(fun) - excinfo.traceback = excinfo.traceback.cut( - path=code.path, firstlineno=code.firstlineno) - return Outcome(excinfo=excinfo, setupfailure=False) Modified: py/branch/distributed/py/test/rsession/hostmanage.py ============================================================================== --- py/branch/distributed/py/test/rsession/hostmanage.py (original) +++ py/branch/distributed/py/test/rsession/hostmanage.py Thu Aug 24 16:05:13 2006 @@ -2,7 +2,7 @@ import py import thread, threading from py.__.test.rsession.master import ( - setup_slave, bin_rsync, MasterNode, dispatch_loop) + setup_slave, MasterNode, dispatch_loop) from py.__.test.rsession import report def init_hosts(reporter, sshhosts, relpath, pkgdir): @@ -53,3 +53,11 @@ report.wrapcall(reporter, channel.waitclose) channel.gateway.exit() +def bin_rsync(source, sshaddress, destpath): + _rsync = py.path.local.sysfind("rsync") + assert destpath + _rsync.sysexec("-az", "--delete-excluded", + "--delete", "--exclude=.svn/", "--exclude=*.pyc", + str(source) + "/", sshaddress + ":" + str(destpath)) + + Modified: py/branch/distributed/py/test/rsession/master.py ============================================================================== --- py/branch/distributed/py/test/rsession/master.py (original) +++ py/branch/distributed/py/test/rsession/master.py Thu Aug 24 16:05:13 2006 @@ -43,14 +43,11 @@ break waiter() -def bin_rsync(source, sshaddress, destpath): - _rsync = py.path.local.sysfind("rsync") - assert destpath - _rsync.sysexec("-az", "--delete", "--exclude=.svn", - str(source), sshaddress + ":" + str(destpath)) - def setup_slave(gateway, pkgpath): from py.__.test.rsession import slave - ch = gateway.remote_exec(py.code.Source(slave.setup, "setup()")) + import os + ch = gateway.remote_exec(str(py.code.Source(slave.setup, "setup()"))) + if hasattr(gateway, 'sshaddress'): + assert not os.path.isabs(pkgpath) ch.send(str(pkgpath)) return ch Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Thu Aug 24 16:05:13 2006 @@ -10,7 +10,7 @@ from py.__.test.rsession import report from py.__.test.rsession.master import ( - setup_slave, bin_rsync, MasterNode, dispatch_loop) + setup_slave, MasterNode, dispatch_loop) from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts class Reporter(object): Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Thu Aug 24 16:05:13 2006 @@ -3,7 +3,7 @@ """ import py -from py.__.test.rsession.executor import Executor, RunExecutor +from py.__.test.rsession.executor import RunExecutor from py.__.test.rsession.outcome import Outcome class SlaveNode(object): @@ -23,6 +23,9 @@ return outcome.make_repr() def slave_main(receive, send, path): + import os + assert os.path.exists(path) + path = os.path.abspath(path) rootcol = py.test.collect.Directory(path) node = SlaveNode(rootcol) while 1: Modified: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_executor.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Thu Aug 24 16:05:13 2006 @@ -2,21 +2,21 @@ import py import example1 -from py.__.test.rsession.executor import Executor, RunExecutor +from py.__.test.rsession.executor import RunExecutor -def test_executor_passing_function(): +def XXXtest_executor_passing_function(): ex = Executor(example1.f1) outcome = ex.execute() assert outcome.passed -def test_executor_raising_function(): +def XXXtest_executor_raising_function(): ex = Executor(example1.g1) outcome = ex.execute() assert not outcome.passed excinfo = outcome.excinfo assert excinfo.type == ValueError -def test_executor_traceback(): +def XXXtest_executor_traceback(): ex = Executor(example1.g1) outcome = ex.execute() excinfo = outcome.excinfo Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Thu Aug 24 16:05:13 2006 @@ -5,7 +5,7 @@ import py from py.__.test.rsession import report from py.__.test.rsession.rsession import RSession -from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts +from py.__.test.rsession.hostmanage import bin_rsync, init_hosts, teardown_hosts from py.__.test.rsession.testing.test_slave import (funcfail_spec, funcpass_spec, funcskip_spec) @@ -57,6 +57,39 @@ py.test.skip("no test distribution ssh hosts specified") cls.hosts = option.disthosts.split(",") + def test_rsync_does_what_it_should(self): + host = self.hosts[0] + gw = py.execnet.SshGateway(host) + channel = gw.remote_exec(""" + import tempfile + tmp = tempfile.mkdtemp() + try: + channel.send(tmp) + channel.receive() # sync + import os + p = os.path.join(tmp, "a") + assert os.path.exists(p) + p2 = os.path.join(p, "__init__.py") + assert os.path.exists(p2) + p3 = os.path.join(p, "x.pyc") + assert not os.path.exists(p3) + p4 = os.path.join(p, "__init__.pyc") + assert not os.path.exists(p4) + channel.send("ok") + finally: + import shutil + shutil.rmtree(tmp) + """) + tmpdir = py.test.ensuretemp("rsynctest") + tmpdir.ensure("a", "__init__.py") + tmpdir.ensure("a", "no.pyc") + tmpdir.ensure("a", "__init__.pyc") + remote_tmpdir = channel.receive() + bin_rsync(tmpdir, host, remote_tmpdir) + channel.send(None) + res = channel.receive() + assert res == "ok" + def test_example_distribution(self): # XXX find a better way for the below tmpdir = py.path.local(py.__file__).dirpath().dirpath() @@ -97,9 +130,9 @@ assert failevents[1].outcome.excinfo.typename == 'ValueError' assert failevents[1].outcome.excinfo.value == '23' tb = failevents[2].outcome.excinfo.traceback - assert tb[0].path.find("test_one") != -1 - assert tb[0].source.find("test_4") != -1 assert failevents[2].outcome.excinfo.typename == 'TypeError' + assert tb[0].path.find("executor") != -1 + assert tb[0].source.find("execute") != -1 def test_setup_teardown_ssh(self): hosts = self.hosts From hpk at codespeak.net Thu Aug 24 16:12:08 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 16:12:08 +0200 (CEST) Subject: [py-svn] r31609 - py/branch/distributed/py/test/rsession Message-ID: <20060824141208.BC09210060@code0.codespeak.net> Author: hpk Date: Thu Aug 24 16:12:06 2006 New Revision: 31609 Modified: py/branch/distributed/py/test/rsession/rsession.py Log: somewhat better defaults for reporting Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Thu Aug 24 16:12:06 2006 @@ -27,22 +27,23 @@ print excinfo def report_unknown(self, what): - print "Unknown report: %s" % what + pass # print "Unknown report: %s" % what - def report_SendItem(self, item): - print "Sending %s to %s" % (item.item, item.channel.gateway.sshaddress) + #def report_SendItem(self, item): + # print "Sending %s to %s" % (item.item, item.channel.gateway.sshaddress) def report_ReceivedItemOutcome(self, event): - sshhost = event.channel.gateway.sshaddress if event.outcome.passed: - print "%s passed:%s" % (event.item, sshhost) + status = "PASSED" elif event.outcome.skipped: - print "%s skipped:%s" % (event.item, sshhost) + status = "SKIPPED" else: - print "%s FAILED: %s" % (event.item, sshhost) + status = "FAILED" excinfo = event.outcome.excinfo - print "".join(excinfo.traceback) - print "%s: %s" % (excinfo.typename, excinfo.value) + status += ": %s -> %s" % (excinfo.typename, excinfo.value) + status += "\n" + "".join(map(str, excinfo.traceback)) + sshhost = event.channel.gateway.sshaddress + print "%10s: %s %s" %(sshhost[:10], event.item.name, status) class RSession(object): """ From hpk at codespeak.net Thu Aug 24 16:16:41 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 16:16:41 +0200 (CEST) Subject: [py-svn] r31610 - py/branch/distributed/py/test/rsession Message-ID: <20060824141641.CB95310060@code0.codespeak.net> Author: hpk Date: Thu Aug 24 16:16:39 2006 New Revision: 31610 Modified: py/branch/distributed/py/test/rsession/rsession.py Log: slightly refined reporting Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Thu Aug 24 16:16:39 2006 @@ -14,6 +14,9 @@ from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts class Reporter(object): + def __init__(self, config): + self.config = config + def report(self, what): repfun = getattr(self, "report_" + what.__class__.__name__, self.report_unknown) @@ -27,23 +30,27 @@ print excinfo def report_unknown(self, what): - pass # print "Unknown report: %s" % what + if self.config.option.verbose: + print "Unknown report: %s" % what - #def report_SendItem(self, item): - # print "Sending %s to %s" % (item.item, item.channel.gateway.sshaddress) + def report_SendItem(self, item): + if self.config.option.verbose: + print "Sending %s to %s" % (item.item, + item.channel.gateway.sshaddress) def report_ReceivedItemOutcome(self, event): if event.outcome.passed: - status = "PASSED" + status = "PASSED " elif event.outcome.skipped: status = "SKIPPED" else: - status = "FAILED" + status = "FAILED " excinfo = event.outcome.excinfo status += ": %s -> %s" % (excinfo.typename, excinfo.value) status += "\n" + "".join(map(str, excinfo.traceback)) sshhost = event.channel.gateway.sshaddress - print "%10s: %s %s" %(sshhost[:10], event.item.name, status) + itempath = " ".join(event.item.listnames()[1:]) + print "%10s: %s %s" %(sshhost[:10], status, itempath) class RSession(object): """ @@ -63,7 +70,7 @@ sshhosts = self.config.getinitialvalue("disthosts") destrelpath = "pytestcache" if reporter is None: - reporter = Reporter().report + reporter = Reporter(self.config).report nodes = init_hosts(reporter, sshhosts, destrelpath, pkgdir) def itemgen(): From fijal at codespeak.net Thu Aug 24 16:32:25 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 24 Aug 2006 16:32:25 +0200 (CEST) Subject: [py-svn] r31611 - py/branch/distributed/py/test/rsession Message-ID: <20060824143225.4209710060@code0.codespeak.net> Author: fijal Date: Thu Aug 24 16:32:24 2006 New Revision: 31611 Modified: py/branch/distributed/py/test/rsession/hostmanage.py Log: Fixed multiple running of the same hosts Modified: py/branch/distributed/py/test/rsession/hostmanage.py ============================================================================== --- py/branch/distributed/py/test/rsession/hostmanage.py (original) +++ py/branch/distributed/py/test/rsession/hostmanage.py Thu Aug 24 16:32:24 2006 @@ -13,10 +13,19 @@ nodes = [] nodes_lock = threading.Condition() exc_info = [None] + rsync_hosts = {} def host_init(host): try: - report.wrapcall(reporter, bin_rsync, + nodes_lock.acquire() + if host in rsync_hosts: + do_rsync = False + else: + do_rsync = True + rsync_hosts[host] = True + nodes_lock.release() + if do_rsync: + report.wrapcall(reporter, bin_rsync, str(pkgdir.dirpath())+"/", host, relpath) gw = py.execnet.SshGateway(host) ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) From fijal at codespeak.net Thu Aug 24 17:30:03 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 24 Aug 2006 17:30:03 +0200 (CEST) Subject: [py-svn] r31612 - py/branch/distributed/py/test/rsession Message-ID: <20060824153003.99A351006C@code0.codespeak.net> Author: fijal Date: Thu Aug 24 17:30:01 2006 New Revision: 31612 Modified: py/branch/distributed/py/test/rsession/hostmanage.py Log: (hpk, fijal) - Made hostinit one after another. Modified: py/branch/distributed/py/test/rsession/hostmanage.py ============================================================================== --- py/branch/distributed/py/test/rsession/hostmanage.py (original) +++ py/branch/distributed/py/test/rsession/hostmanage.py Thu Aug 24 17:30:01 2006 @@ -10,48 +10,21 @@ "%s probably wrong" %(pkgdir,)) assert relpath, relpath assert not os.path.isabs(relpath), relpath + nodes = [] nodes_lock = threading.Condition() exc_info = [None] - rsync_hosts = {} - + def host_init(host): - try: - nodes_lock.acquire() - if host in rsync_hosts: - do_rsync = False - else: - do_rsync = True - rsync_hosts[host] = True - nodes_lock.release() - if do_rsync: - report.wrapcall(reporter, bin_rsync, - str(pkgdir.dirpath())+"/", host, relpath) - gw = py.execnet.SshGateway(host) - ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) - nodes_lock.acquire() - try: - nodes.append(MasterNode(ch, reporter)) - nodes_lock.notify() - finally: - nodes_lock.release() - except: - exc_info[0] = sys.exc_info() - + report.wrapcall(reporter, bin_rsync, + str(pkgdir.dirpath())+"/", host, relpath) + gw = py.execnet.SshGateway(host) + ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) + nodes.append(MasterNode(ch, reporter)) + for host in sshhosts: - thread.start_new_thread(host_init, (host,)) - - while len(nodes) != len(sshhosts): - if exc_info != [None]: - exc, data, tb = exc_info[0] - raise exc, data, tb - nodes_lock.acquire() - try: - nodes_lock.wait(1) - finally: - nodes_lock.release() + host_init(host) - return nodes def teardown_hosts(reporter, channels): From hpk at codespeak.net Thu Aug 24 17:34:03 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 24 Aug 2006 17:34:03 +0200 (CEST) Subject: [py-svn] r31613 - py/branch/distributed Message-ID: <20060824153403.2E6ED1006C@code0.codespeak.net> Author: hpk Date: Thu Aug 24 17:34:01 2006 New Revision: 31613 Modified: py/branch/distributed/TODO Log: (fijal, hpk) listed a number of pending items for distributed testing Modified: py/branch/distributed/TODO ============================================================================== --- py/branch/distributed/TODO (original) +++ py/branch/distributed/TODO Thu Aug 24 17:34:01 2006 @@ -4,39 +4,31 @@ 1st. Milestone ----------------- +DONE * use "/usr/bin/rsync" to sync to remote * master does collection of items * "leaf" items are asynchronously send to "Executors" (running on nodes) * super-simple reporting -# running at MASTER: -channels = [] -for gw in mygateways: - c = gw.remote_exec(executor_protocol) - c.setcallback(lambda x: report_queue.put) - channels.append(c) -try: - # wait for completion -finally: - for c in channels: - c.send(None) - c.close() - -# dispatching thread: -while 1: - for node in nodes: - if len(node.pending) < MAXTASKS_PER_NODE: - item = itemgenerator.next() - if shouldstop: - break - node.send(item) - - -# reporting thread -while 1: - if shouldstop: - break - report_event = report_queue.get() - print report_event +open issues: +* sending cmdline and other options to remote places +* capturing stdout/stderr remotely (and sending it back) +* noticing hanging tests/hosts +* reacting to hanging tests/hosts (probably first just remove + the masternode from the dispatching list and reschedule pending + tests to a different host and report about all this) +* more events, at least: + - start testing + - finish testing + - setup/teardown of hosts + - segfaults + - rsyncing (after 1-n rsync) +* nice hostmanagement +* 1-n rsync +* improved reporting, e.g. + report which packages are used from where on the remote side +* lazy remote py lib import? +* --pdb -> remote screen sessions +* later: caring for specifying python versions / hosts From fijal at codespeak.net Fri Aug 25 16:52:25 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 Aug 2006 16:52:25 +0200 (CEST) Subject: [py-svn] r31643 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060825145225.4A5A010087@code0.codespeak.net> Author: fijal Date: Fri Aug 25 16:52:21 2006 New Revision: 31643 Modified: py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/hostmanage.py py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/report.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_report.py Log: Fixed reporting a little bit, added displaying of skipped items (which were skipped on the remote side, actually tests skipped on the master side are just ignored). Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Fri Aug 25 16:52:21 2006 @@ -15,8 +15,8 @@ try: self.item.run() return Outcome() - except py.test.Item.Skipped: - return Outcome(skipped=True) + except py.test.Item.Skipped, e: + return Outcome(skipped=str(e)) except: excinfo = py.code.ExceptionInfo() if isinstance(self.item, py.test.Function): Modified: py/branch/distributed/py/test/rsession/hostmanage.py ============================================================================== --- py/branch/distributed/py/test/rsession/hostmanage.py (original) +++ py/branch/distributed/py/test/rsession/hostmanage.py Fri Aug 25 16:52:21 2006 @@ -21,6 +21,7 @@ gw = py.execnet.SshGateway(host) ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) nodes.append(MasterNode(ch, reporter)) + reporter(report.HostReady(ch.gateway.sshaddress)) for host in sshhosts: host_init(host) Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Fri Aug 25 16:52:21 2006 @@ -9,12 +9,12 @@ import sys class Outcome(object): - def __init__(self, setupfailure=False, excinfo=None, skipped=False, + def __init__(self, setupfailure=False, excinfo=None, skipped=None, is_critical=False): self.passed = not excinfo and not skipped self.skipped = skipped self.setupfailure = setupfailure - self.excinfo = excinfo + self.excinfo = excinfo self.is_critical = is_critical assert bool(self.passed) + bool(excinfo) + bool(skipped) == 1 Modified: py/branch/distributed/py/test/rsession/report.py ============================================================================== --- py/branch/distributed/py/test/rsession/report.py (original) +++ py/branch/distributed/py/test/rsession/report.py Fri Aug 25 16:52:21 2006 @@ -1,8 +1,8 @@ - """ Reporter classes for showing asynchronous and synchronous status events """ import py +import time def basic_report(msg_type, message): print msg_type, message @@ -69,5 +69,18 @@ class CallFinish(CallEvent): pass +class HostReady(ReportEvent): + def __init__(self, hostname): + self.hostname = hostname + +class TestStarted(ReportEvent): + def __init__(self, hosts): + self.hosts = hosts + self.timestart = time.time() + +class TestFinished(ReportEvent): + def __init__(self): + self.timeend = time.time() + # tryiter, main dispatch loop, something else (setup-teardown stuff) # Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Fri Aug 25 16:52:21 2006 @@ -13,9 +13,17 @@ setup_slave, MasterNode, dispatch_loop) from py.__.test.rsession.hostmanage import init_hosts, teardown_hosts +from py.__.test.terminal.out import getout + class Reporter(object): - def __init__(self, config): + def __init__(self, config, hosts): self.config = config + self.failed_tests_outcome = [] + self.skipped_tests_outcome = [] + self.out = getout(py.std.sys.stdout) + self.failed = dict([(host, 0) for host in hosts]) + self.skipped = dict([(host, 0) for host in hosts]) + self.passed = dict([(host, 0) for host in hosts]) def report(self, what): repfun = getattr(self, "report_" + what.__class__.__name__, @@ -27,6 +35,8 @@ except: print "Internal reporting problem" excinfo = py.code.ExceptionInfo() + for i in excinfo.traceback: + print str(i)[2:-1] print excinfo def report_unknown(self, what): @@ -38,16 +48,95 @@ print "Sending %s to %s" % (item.item, item.channel.gateway.sshaddress) + def report_HostReady(self, item): + print "%10s: READY" % item.hostname[:10] + + def report_TestStarted(self, item): + txt = " Test started, hosts: %s " % ", ".join(item.hosts) + self.out.sep("=", txt) + self.timestart = item.timestart + + def report_TestFinished(self, item): + assert hasattr(self, 'timestart') + self.timeend = item.timeend + self.skips() + self.failures() + self.summary() + + def failures(self): + pass + + def skips(self): + texts = {} + for event in self.skipped_tests_outcome: + colitem = event.item + outcome = event.outcome + text = outcome.skipped + itemname = event.channel.gateway.sshaddress + ":" + \ + "/".join(colitem.listnames()) + if text not in texts: + texts[text] = [itemname] + else: + texts[text].append(itemname) + + if texts: + self.out.line() + self.out.sep('_', 'reasons for skipped tests') + for text, items in texts.items(): + for item in items: + self.out.line('Skipped in %s' % item) + self.out.line("reason: %s" % text) +## raisingtb = self.getlastvisible(outcome.excinfo.traceback) +## fn = raisingtb.frame.code.path +## lineno = raisingtb.lineno +## d = texts.setdefault(outcome.excinfo.exconly(), {}) +## d[(fn,lineno)] = outcome +## +## if texts: +## self.out.line() +## self.out.sep('_', 'reasons for skipped tests') +## for text, dict in texts.items(): +## for (fn, lineno), outcome in dict.items(): +## self.out.line('Skipped in %s:%d' %(fn, lineno+1)) +## self.out.line("reason: %s" % text) +## self.out.line() + + + def summary(self): + def gather(dic): + total = 0 + for key, val in dic.iteritems(): + total += val + return total + + def create_str(name, count): + if count: + return ", %d %s" % (count, name) + return "" + + total_passed = gather(self.passed) + total_failed = gather(self.failed) + total_skipped = gather(self.skipped) + total = total_passed + total_failed + total_skipped + skipped_str = create_str("skipped", total_skipped) + failed_str = create_str("failed", total_failed) + self.out.sep("=", " %d test run%s%s in %.2fs " % + (total, skipped_str, failed_str, self.timeend - self.timestart)) + def report_ReceivedItemOutcome(self, event): + host = event.channel.gateway.sshaddress if event.outcome.passed: status = "PASSED " + self.passed[host] += 1 elif event.outcome.skipped: status = "SKIPPED" + self.skipped_tests_outcome.append(event) + self.skipped[host] += 1 else: status = "FAILED " - excinfo = event.outcome.excinfo - status += ": %s -> %s" % (excinfo.typename, excinfo.value) - status += "\n" + "".join(map(str, excinfo.traceback)) + self.failed[host] += 1 + self.failed_tests_outcome.append(event) + # we'll take care of them later sshhost = event.channel.gateway.sshaddress itempath = " ".join(event.item.listnames()[1:]) print "%10s: %s %s" %(sshhost[:10], status, itempath) @@ -64,13 +153,14 @@ """ main loop for running tests. """ if not args: args = [py.path.local()] + sshhosts = self.config.getinitialvalue("disthosts") + if reporter is None: + reporter = Reporter(self.config, sshhosts).report + reporter(report.TestStarted(sshhosts)) pkgdir = self.getpkgdir(args[0]) colitems = self.make_colitems(args, baseon=pkgdir.dirpath()) - sshhosts = self.config.getinitialvalue("disthosts") destrelpath = "pytestcache" - if reporter is None: - reporter = Reporter(self.config).report nodes = init_hosts(reporter, sshhosts, destrelpath, pkgdir) def itemgen(): @@ -83,6 +173,7 @@ # list(itemgenerator)]) dispatch_loop(nodes, itemgenerator, lambda : False) teardown_hosts(reporter, [node.channel for node in nodes]) + reporter(report.TestFinished()) def make_colitems(paths, baseon): # we presume that from the base we can simply get to Modified: py/branch/distributed/py/test/rsession/testing/test_report.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_report.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_report.py Fri Aug 25 16:52:21 2006 @@ -23,3 +23,14 @@ assert len(l) == 2 assert isinstance(l[0], report.CallStart) assert isinstance(l[1], report.CallException) + +def test_reporter_methods_sanity(): + """ Checks if all the methods of reporter are sane + """ + from py.__.test.rsession.rsession import Reporter + from py.__.test.rsession import report + + for method in dir(Reporter): + + if method.startswith("report_") and method != "report_unknown": + assert method[len('report_'):] in report.__dict__ From fijal at codespeak.net Fri Aug 25 19:39:47 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 Aug 2006 19:39:47 +0200 (CEST) Subject: [py-svn] r31657 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060825173947.39DA310084@code0.codespeak.net> Author: fijal Date: Fri Aug 25 19:39:45 2006 New Revision: 31657 Modified: py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/testing/test_outcome.py Log: Added relative line control, for showing where exactly something went wrong. Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Fri Aug 25 19:39:45 2006 @@ -27,12 +27,13 @@ def traceback_entry_repr(self, tb_entry): lineno = tb_entry.lineno + relline = tb_entry.frame.code.firstlineno - lineno + 1 path = str(tb_entry.path) try: source = str(tb_entry.getsource()) except: source = "" - return (lineno, source, path) + return (relline, lineno, source, path) def make_repr(self): return (self.passed, self.setupfailure, @@ -41,7 +42,8 @@ class TracebackEntryRepr(object): def __init__(self, tbentry): - lineno, self.source, self.path = tbentry + relline, lineno, self.source, self.path = tbentry + self.relline = int(relline) self.lineno = int(lineno) def __repr__(self): Modified: py/branch/distributed/py/test/rsession/testing/test_outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_outcome.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_outcome.py Fri Aug 25 19:39:45 2006 @@ -34,7 +34,10 @@ myfile = py.magic.autopath() assert excinfo.traceback[3].path == myfile assert excinfo.traceback[3].lineno == f1.func_code.co_firstlineno + assert excinfo.traceback[3].relline == 0 assert excinfo.traceback[2].path == myfile assert excinfo.traceback[2].lineno == f2.func_code.co_firstlineno + assert excinfo.traceback[2].relline == 0 assert excinfo.traceback[1].path == myfile assert excinfo.traceback[1].lineno == f3.func_code.co_firstlineno + assert excinfo.traceback[1].relline == 0 From fijal at codespeak.net Fri Aug 25 19:56:59 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 Aug 2006 19:56:59 +0200 (CEST) Subject: [py-svn] r31658 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060825175659.CB13F10084@code0.codespeak.net> Author: fijal Date: Fri Aug 25 19:56:57 2006 New Revision: 31658 Modified: py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_outcome.py py/branch/distributed/py/test/rsession/testing/test_report.py Log: Added some reporting. This is totally different from our stuff from terminal.py, because information that we have is totally different as well. We need to accept different options though. Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Fri Aug 25 19:56:57 2006 @@ -27,7 +27,7 @@ def traceback_entry_repr(self, tb_entry): lineno = tb_entry.lineno - relline = tb_entry.frame.code.firstlineno - lineno + 1 + relline = lineno - tb_entry.frame.code.firstlineno path = str(tb_entry.path) try: source = str(tb_entry.getsource()) Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Fri Aug 25 19:56:57 2006 @@ -64,8 +64,39 @@ self.summary() def failures(self): - pass + self.out.sep("=", " FAILURES ") + for event in self.failed_tests_outcome: + host = event.channel.gateway.sshaddress + self.out.sep('_', "%s on %s" % + (" ".join(event.item.listnames()), host)) + self.repr_failure(event.item, event.outcome) + def repr_failure(self, item, outcome): + excinfo = outcome.excinfo + traceback = excinfo.traceback + if item and not self.config.option.fulltrace: + path, firstlineno = item.getpathlineno() + if not traceback: + self.out.line("empty traceback from item %r" % (item,)) + return + #handler = getattr(self, 'repr_failure_tb%s' % self.config.option.tbstyle) + handler = self.repr_failure_tblong + handler(item, excinfo, traceback) + + def repr_failure_tblong(self, item, excinfo, traceback): + for index, entry in py.builtin.enumerate(traceback): + self.out.sep('-') + self.out.line("%s: %s" % (entry.path, entry.lineno)) + self.repr_source(entry.relline, entry.source) + self.out.line("%s: %s" % (excinfo.typename, excinfo.value)) + + def repr_source(self, relline, source): + for num, line in enumerate(source.split("\n")): + if num == relline: + self.out.line(">>>>" + line) + else: + self.out.line(" " + line) + def skips(self): texts = {} for event in self.skipped_tests_outcome: @@ -86,21 +117,6 @@ for item in items: self.out.line('Skipped in %s' % item) self.out.line("reason: %s" % text) -## raisingtb = self.getlastvisible(outcome.excinfo.traceback) -## fn = raisingtb.frame.code.path -## lineno = raisingtb.lineno -## d = texts.setdefault(outcome.excinfo.exconly(), {}) -## d[(fn,lineno)] = outcome -## -## if texts: -## self.out.line() -## self.out.sep('_', 'reasons for skipped tests') -## for text, dict in texts.items(): -## for (fn, lineno), outcome in dict.items(): -## self.out.line('Skipped in %s:%d' %(fn, lineno+1)) -## self.out.line("reason: %s" % text) -## self.out.line() - def summary(self): def gather(dic): Modified: py/branch/distributed/py/test/rsession/testing/test_outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_outcome.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_outcome.py Fri Aug 25 19:56:57 2006 @@ -10,6 +10,10 @@ assert r.is_critical def f1(): + 1 + 2 + 3 + 4 raise ValueError(42) def f2(): @@ -17,7 +21,7 @@ def f3(): f2() - + def test_exception_info_repr(): try: f3() @@ -33,11 +37,14 @@ assert len(excinfo.traceback) == 4 myfile = py.magic.autopath() assert excinfo.traceback[3].path == myfile - assert excinfo.traceback[3].lineno == f1.func_code.co_firstlineno - assert excinfo.traceback[3].relline == 0 + assert excinfo.traceback[3].lineno == f1.func_code.co_firstlineno + 4 + assert excinfo.traceback[3].relline == 5 assert excinfo.traceback[2].path == myfile assert excinfo.traceback[2].lineno == f2.func_code.co_firstlineno - assert excinfo.traceback[2].relline == 0 + assert excinfo.traceback[2].relline == 1 assert excinfo.traceback[1].path == myfile assert excinfo.traceback[1].lineno == f3.func_code.co_firstlineno - assert excinfo.traceback[1].relline == 0 + assert excinfo.traceback[1].relline == 1 + +#def test_f3(): +# f3() Modified: py/branch/distributed/py/test/rsession/testing/test_report.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_report.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_report.py Fri Aug 25 19:56:57 2006 @@ -34,3 +34,10 @@ if method.startswith("report_") and method != "report_unknown": assert method[len('report_'):] in report.__dict__ + +def f(): + sad + +#def test_fail(): +# f() + From fijal at codespeak.net Sat Aug 26 14:53:12 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 Aug 2006 14:53:12 +0200 (CEST) Subject: [py-svn] r31679 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060826125312.A55A210089@code0.codespeak.net> Author: fijal Date: Sat Aug 26 14:53:09 2006 New Revision: 31679 Added: py/branch/distributed/py/test/rsession/box.py (contents, props changed) py/branch/distributed/py/test/rsession/testing/test_boxing.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/testing/test_report.py Log: Added basic boxing stuff. Added: py/branch/distributed/py/test/rsession/box.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/box.py Sat Aug 26 14:53:09 2006 @@ -0,0 +1,57 @@ + +""" boxing - wrapping process with another process so we can run +a process inside and see if it crashes +""" + +import py +import os +import sys + +PYTESTSTDOUT = "pyteststdout" +PYTESTSTDERR = "pyteststderr" + +class Box(object): + def __init__(self, fun): + self.fun = fun + + def run(self): + # What to do: + # XXX: Creates it in a current dir, and just open it, does not + # seem to be sane idea + os.mkfifo(PYTESTSTDOUT) + os.mkfifo(PYTESTSTDERR) + pid = os.fork() + if pid: + self.parent() + else: + self.children() + os._exit(0) + + def children(self): + # right now we need to call a function, but first we need to + # map all IO that might happen + fdstdout = os.open(PYTESTSTDOUT, os.O_WRONLY) + fdstderr = os.open(PYTESTSTDERR, os.O_WRONLY) + os.dup2(fdstdout, 1) + os.dup2(fdstderr, 2) + os.close(0) + self.fun() + #sys.exit(0) + + def parent(self): + self.fdstdout = open(PYTESTSTDOUT, "r") + self.fdstderr = open(PYTESTSTDERR, "r") + pid, exitstat = os.wait() + self.signal = exitstat & 0x7f + self.exitstat = exitstat & 0xff00 + self.stdout = self.fdstdout.read() + self.stderr = self.fdstderr.read() + self.fdstdout.close() + self.fdstderr.close() + + def __del__(self): + try: + os.unlink(PYTESTSTDOUT) + os.unlink(PYTESTSTDERR) + except: + pass Added: py/branch/distributed/py/test/rsession/testing/test_boxing.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/test_boxing.py Sat Aug 26 14:53:09 2006 @@ -0,0 +1,22 @@ + +""" test boxing functionality +""" + +from py.__.test.rsession.box import Box +from py.__.test.rsession.testing import example2 + +def test_basic_boxing(): + b = Box(example2.boxf1) + b.run() + assert b.stdout == "some out\n" + assert b.stderr == "some err\n" + assert b.exitstat == 0 + assert b.signal == 0 + +def test_boxing_on_fds(): + b = Box(example2.boxf2) + b.run() + assert b.stdout == "someout" + assert b.stderr == "someerr" + assert b.exitstat == 0 + assert b.signal == 0 Modified: py/branch/distributed/py/test/rsession/testing/test_report.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_report.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_report.py Sat Aug 26 14:53:09 2006 @@ -34,10 +34,3 @@ if method.startswith("report_") and method != "report_unknown": assert method[len('report_'):] in report.__dict__ - -def f(): - sad - -#def test_fail(): -# f() - From fijal at codespeak.net Sat Aug 26 16:37:58 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 Aug 2006 16:37:58 +0200 (CEST) Subject: [py-svn] r31683 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060826143758.527E010089@code0.codespeak.net> Author: fijal Date: Sat Aug 26 16:37:54 2006 New Revision: 31683 Added: py/branch/distributed/py/test/rsession/testing/example2.py (contents, props changed) Modified: py/branch/distributed/py/test/rsession/box.py py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/outcome.py py/branch/distributed/py/test/rsession/slave.py py/branch/distributed/py/test/rsession/testing/test_boxing.py py/branch/distributed/py/test/rsession/testing/test_executor.py py/branch/distributed/py/test/rsession/testing/test_slave.py Log: Added wrapping executor which is now default. It may box execution stuff, so on the local side you may see stdout, stderr and eventual segfaults. Modified: py/branch/distributed/py/test/rsession/box.py ============================================================================== --- py/branch/distributed/py/test/rsession/box.py (original) +++ py/branch/distributed/py/test/rsession/box.py Sat Aug 26 16:37:54 2006 @@ -6,52 +6,79 @@ import py import os import sys +import marshal PYTESTSTDOUT = "pyteststdout" PYTESTSTDERR = "pyteststderr" +PYTESTRETVAL = "pytestretval" class Box(object): - def __init__(self, fun): + def __init__(self, fun, args = [], kwargs = {}): self.fun = fun + self.args = args + self.kwargs = kwargs def run(self): # What to do: # XXX: Creates it in a current dir, and just open it, does not # seem to be sane idea + try: + self.clear() + except OSError: + pass os.mkfifo(PYTESTSTDOUT) os.mkfifo(PYTESTSTDERR) + os.mkfifo(PYTESTRETVAL) pid = os.fork() if pid: self.parent() else: - self.children() + try: + outcome = self.children() + except: + excinfo = py.code.ExceptionInfo() + print "Internal box error" + for i in excinfo.traceback: + print str(i)[2:-1] + print excinfo + os._exit(1) + os._exit(0) def children(self): # right now we need to call a function, but first we need to # map all IO that might happen + os.close(1) + os.close(2) fdstdout = os.open(PYTESTSTDOUT, os.O_WRONLY) fdstderr = os.open(PYTESTSTDERR, os.O_WRONLY) + retvalf = open(PYTESTRETVAL, "w") os.dup2(fdstdout, 1) os.dup2(fdstderr, 2) os.close(0) - self.fun() - #sys.exit(0) + retval = self.fun(*self.args, **self.kwargs) + retvalf.write(marshal.dumps(retval)) + retvalf.close() def parent(self): self.fdstdout = open(PYTESTSTDOUT, "r") self.fdstderr = open(PYTESTSTDERR, "r") + retval = open(PYTESTRETVAL, "r") pid, exitstat = os.wait() self.signal = exitstat & 0x7f self.exitstat = exitstat & 0xff00 + if not exitstat: + self.retval = marshal.loads(retval.read()) + retval.close() + else: + self.retval = None self.stdout = self.fdstdout.read() self.stderr = self.fdstderr.read() self.fdstdout.close() self.fdstderr.close() + self.clear() - def __del__(self): - try: - os.unlink(PYTESTSTDOUT) - os.unlink(PYTESTSTDERR) - except: - pass + def clear(self): + os.unlink(PYTESTSTDOUT) + os.unlink(PYTESTSTDERR) + os.unlink(PYTESTRETVAL) Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Sat Aug 26 16:37:54 2006 @@ -4,10 +4,13 @@ import py from py.__.test.rsession.outcome import Outcome +from py.__.test.rsession.box import Box class RunExecutor(object): """ Same as in executor, but just running run """ + wraps = False + def __init__(self, item): self.item = item @@ -26,7 +29,25 @@ path=code.path, firstlineno=code.firstlineno) return Outcome(excinfo=excinfo, setupfailure=False) - +class BoxExecutor(RunExecutor): + """ Same as run executor, but boxes test instead + """ + wraps = True + + def execute(self): + def fun(): + outcome = RunExecutor.execute(self) + return outcome.make_repr() + b = Box(fun) + b.run() + if b.retval is not None: + passed, setupfailure, excinfo, skipped, critical, _, _, _ = b.retval + return (passed, setupfailure, excinfo, skipped, critical, 0, + b.stdout, b.stderr) + else: + if b.signal == 0: + print b.stdout + return (False, False, None, False, False, b.signal, b.stdout, b.stderr) # XXX old code class XXXExecutor(object): Modified: py/branch/distributed/py/test/rsession/outcome.py ============================================================================== --- py/branch/distributed/py/test/rsession/outcome.py (original) +++ py/branch/distributed/py/test/rsession/outcome.py Sat Aug 26 16:37:54 2006 @@ -38,7 +38,7 @@ def make_repr(self): return (self.passed, self.setupfailure, self.make_excinfo_repr(), - self.skipped, self.is_critical) + self.skipped, self.is_critical, 0, "", "") class TracebackEntryRepr(object): def __init__(self, tbentry): @@ -62,7 +62,7 @@ class ReprOutcome(object): def __init__(self, repr_tuple): (self.passed, self.setupfailure, excinfo, self.skipped, - self.is_critical) = repr_tuple + self.is_critical, self.signal, self.stdout, self.stderr) = repr_tuple if excinfo is None: self.excinfo = None else: @@ -70,5 +70,5 @@ def __repr__(self): l = ["%s=%s" %(x, getattr(self, x)) - for x in "passed skipped setupfailure excinfo".split()] + for x in "signal passed skipped setupfailure excinfo stdout stderr".split()] return "" %(" ".join(l),) Modified: py/branch/distributed/py/test/rsession/slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/slave.py (original) +++ py/branch/distributed/py/test/rsession/slave.py Sat Aug 26 16:37:54 2006 @@ -3,24 +3,30 @@ """ import py -from py.__.test.rsession.executor import RunExecutor +from py.__.test.rsession.executor import RunExecutor, BoxExecutor from py.__.test.rsession.outcome import Outcome class SlaveNode(object): - def __init__(self, rootcollector): + def __init__(self, rootcollector, executor=BoxExecutor): self.rootcollector = rootcollector + self.executor = executor def execute(self, itemspec): item = self.rootcollector.getitembynames(itemspec) #if isinstance(item, py.test.Function): # ex = Executor(item.obj, setup=item.setup) #else: - ex = RunExecutor(item) + ex = self.executor(item) return ex.execute() def run(self, itemspec): + #outcome = self.execute(itemspec) + #return outcome.make_repr() outcome = self.execute(itemspec) - return outcome.make_repr() + if self.executor.wraps: + return outcome + else: + return outcome.make_repr() def slave_main(receive, send, path): import os Added: py/branch/distributed/py/test/rsession/testing/example2.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/testing/example2.py Sat Aug 26 16:37:54 2006 @@ -0,0 +1,19 @@ + +""" some example for running box stuff inside +""" + +import sys +import py, os + +def boxf1(): + print "some out" + print >>sys.stderr, "some err" + return 1 + +def boxf2(): + os.write(1, "someout") + os.write(2, "someerr") + return 2 + +def boxseg(): + os.kill(os.getpid(), 11) Modified: py/branch/distributed/py/test/rsession/testing/test_boxing.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_boxing.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_boxing.py Sat Aug 26 16:37:54 2006 @@ -2,16 +2,22 @@ """ test boxing functionality """ +import py + from py.__.test.rsession.box import Box from py.__.test.rsession.testing import example2 +from py.__.test.rsession.conftest import option def test_basic_boxing(): + if not option.nocapture: + py.test.skip("Interacts with pylib i/o skipping which is bad actually") b = Box(example2.boxf1) b.run() assert b.stdout == "some out\n" assert b.stderr == "some err\n" assert b.exitstat == 0 assert b.signal == 0 + assert b.retval == 1 def test_boxing_on_fds(): b = Box(example2.boxf2) @@ -20,3 +26,11 @@ assert b.stderr == "someerr" assert b.exitstat == 0 assert b.signal == 0 + assert b.retval == 2 + +def test_boxing_signal(): + b = Box(example2.boxseg) + b.run() + assert b.signal == 11 + assert b.retval is None + Modified: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_executor.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Sat Aug 26 16:37:54 2006 @@ -2,7 +2,8 @@ import py import example1 -from py.__.test.rsession.executor import RunExecutor +from py.__.test.rsession.executor import RunExecutor, BoxExecutor +from py.__.test.rsession.outcome import ReprOutcome def XXXtest_executor_passing_function(): ex = Executor(example1.f1) @@ -60,8 +61,26 @@ outcome = ex.execute() assert not outcome.passed - ex = RunExecutor(ItemTestSkipping("fail")) + ex = RunExecutor(ItemTestSkipping("skip")) outcome = ex.execute() assert outcome.skipped assert not outcome.passed assert not outcome.excinfo + +def test_box_executor(): + ex = BoxExecutor(ItemTestPassing("pass")) + outcome_repr = ex.execute() + outcome = ReprOutcome(outcome_repr) + assert outcome.passed + + ex = BoxExecutor(ItemTestFailing("fail")) + outcome_repr = ex.execute() + outcome = ReprOutcome(outcome_repr) + assert not outcome.passed + + ex = BoxExecutor(ItemTestSkipping("skip")) + outcome_repr = ex.execute() + outcome = ReprOutcome(outcome_repr) + assert outcome.skipped + assert not outcome.passed + assert not outcome.excinfo Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Sat Aug 26 16:37:54 2006 @@ -24,10 +24,11 @@ # ---------------------------------------------------------------------- +from py.__.test.rsession.executor import RunExecutor def gettestnode(): rootcol = py.test.collect.Directory(rootdir) - node = SlaveNode(rootcol) + node = SlaveNode(rootcol, executor=RunExecutor) return node def test_slave_run_passing(): From fijal at codespeak.net Sat Aug 26 18:22:32 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 Aug 2006 18:22:32 +0200 (CEST) Subject: [py-svn] r31687 - in py/branch/distributed/py/test/rsession: . testing Message-ID: <20060826162232.E077D10090@code0.codespeak.net> Author: fijal Date: Sat Aug 26 18:22:29 2006 New Revision: 31687 Modified: py/branch/distributed/py/test/rsession/box.py py/branch/distributed/py/test/rsession/executor.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_boxing.py py/branch/distributed/py/test/rsession/testing/test_executor.py py/branch/distributed/py/test/rsession/testing/test_rsession.py py/branch/distributed/py/test/rsession/testing/test_slave.py Log: Added capturing of stdout/stderr. The bad thing about it is that rsession could not run it's tests remotely, cause magic is done twice (and part of magic is to overload actually py.test magic :)) So what we actually can do is to provide some option like im_remote to help this kind of stuff. Modified: py/branch/distributed/py/test/rsession/box.py ============================================================================== --- py/branch/distributed/py/test/rsession/box.py (original) +++ py/branch/distributed/py/test/rsession/box.py Sat Aug 26 18:22:29 2006 @@ -42,20 +42,24 @@ print str(i)[2:-1] print excinfo os._exit(1) - + sys.stdout.flush() + sys.stderr.flush() os._exit(0) + return pid def children(self): # right now we need to call a function, but first we need to # map all IO that might happen - os.close(1) - os.close(2) + # make sure sys.stdout points to file descriptor one fdstdout = os.open(PYTESTSTDOUT, os.O_WRONLY) + if fdstdout != 1: + os.dup2(fdstdout, 1) fdstderr = os.open(PYTESTSTDERR, os.O_WRONLY) + if fdstderr != 2: + os.dup2(fdstderr, 2) + sys.stdout = os.fdopen(1, "w") + sys.stderr = os.fdopen(2, "w") retvalf = open(PYTESTRETVAL, "w") - os.dup2(fdstdout, 1) - os.dup2(fdstderr, 2) - os.close(0) retval = self.fun(*self.args, **self.kwargs) retvalf.write(marshal.dumps(retval)) retvalf.close() @@ -68,17 +72,22 @@ self.signal = exitstat & 0x7f self.exitstat = exitstat & 0xff00 if not exitstat: - self.retval = marshal.loads(retval.read()) + retval_data = retval.read() + self.retval = marshal.loads(retval_data) retval.close() else: self.retval = None - self.stdout = self.fdstdout.read() - self.stderr = self.fdstderr.read() + self.stdoutrepr = self.fdstdout.read() + self.stderrrepr = self.fdstderr.read() self.fdstdout.close() self.fdstderr.close() self.clear() + return self.stdoutrepr, self.stderrrepr def clear(self): - os.unlink(PYTESTSTDOUT) - os.unlink(PYTESTSTDERR) - os.unlink(PYTESTRETVAL) + try: + os.unlink(PYTESTSTDOUT) + os.unlink(PYTESTSTDERR) + os.unlink(PYTESTRETVAL) + except OSError: + pass Modified: py/branch/distributed/py/test/rsession/executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/executor.py (original) +++ py/branch/distributed/py/test/rsession/executor.py Sat Aug 26 18:22:29 2006 @@ -1,7 +1,7 @@ """ Remote executor """ -import py +import py, os from py.__.test.rsession.outcome import Outcome from py.__.test.rsession.box import Box @@ -39,15 +39,14 @@ outcome = RunExecutor.execute(self) return outcome.make_repr() b = Box(fun) - b.run() + pid = b.run() + assert pid if b.retval is not None: passed, setupfailure, excinfo, skipped, critical, _, _, _ = b.retval return (passed, setupfailure, excinfo, skipped, critical, 0, - b.stdout, b.stderr) + b.stdoutrepr, b.stderrrepr) else: - if b.signal == 0: - print b.stdout - return (False, False, None, False, False, b.signal, b.stdout, b.stderr) + return (False, False, None, False, False, b.signal, b.stdoutrepr, b.stderrrepr) # XXX old code class XXXExecutor(object): Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Sat Aug 26 18:22:29 2006 @@ -64,7 +64,8 @@ self.summary() def failures(self): - self.out.sep("=", " FAILURES ") + if self.failed_tests_outcome: + self.out.sep("=", " FAILURES ") for event in self.failed_tests_outcome: host = event.channel.gateway.sshaddress self.out.sep('_', "%s on %s" % @@ -82,6 +83,13 @@ #handler = getattr(self, 'repr_failure_tb%s' % self.config.option.tbstyle) handler = self.repr_failure_tblong handler(item, excinfo, traceback) + if outcome.stdout: + self.out.sep('-', " Captured process stdout: ") + self.out.write(outcome.stdout) + if outcome.stderr: + self.out.sep('-', " Captured process stderr: ") + self.out.write(outcome.stderr) + def repr_failure_tblong(self, item, excinfo, traceback): for index, entry in py.builtin.enumerate(traceback): Modified: py/branch/distributed/py/test/rsession/testing/test_boxing.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_boxing.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_boxing.py Sat Aug 26 18:22:29 2006 @@ -2,19 +2,20 @@ """ test boxing functionality """ -import py +import py, sys from py.__.test.rsession.box import Box from py.__.test.rsession.testing import example2 from py.__.test.rsession.conftest import option def test_basic_boxing(): - if not option.nocapture: - py.test.skip("Interacts with pylib i/o skipping which is bad actually") + # XXX: because we do not have option transfer +## if not hasattr(option, 'nocapture') or not option.nocapture: +## py.test.skip("Interacts with pylib i/o skipping which is bad actually") b = Box(example2.boxf1) b.run() - assert b.stdout == "some out\n" - assert b.stderr == "some err\n" + assert b.stdoutrepr == "some out\n" + assert b.stderrrepr == "some err\n" assert b.exitstat == 0 assert b.signal == 0 assert b.retval == 1 @@ -22,8 +23,8 @@ def test_boxing_on_fds(): b = Box(example2.boxf2) b.run() - assert b.stdout == "someout" - assert b.stderr == "someerr" + assert b.stdoutrepr == "someout" + assert b.stderrrepr == "someerr" assert b.exitstat == 0 assert b.signal == 0 assert b.retval == 2 @@ -33,4 +34,3 @@ b.run() assert b.signal == 11 assert b.retval is None - Modified: py/branch/distributed/py/test/rsession/testing/test_executor.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_executor.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_executor.py Sat Aug 26 18:22:29 2006 @@ -4,7 +4,13 @@ from py.__.test.rsession.executor import RunExecutor, BoxExecutor from py.__.test.rsession.outcome import ReprOutcome +from py.__.test.rsession.testing.test_slave import funcprint_spec, \ + funcprintfail_spec +from py.__.test.rsession.conftest import option +def setup_module(mod): + mod.rootdir = py.path.local(py.__file__).dirpath().dirpath() + def XXXtest_executor_passing_function(): ex = Executor(example1.f1) outcome = ex.execute() @@ -84,3 +90,21 @@ assert outcome.skipped assert not outcome.passed assert not outcome.excinfo + +def test_box_executor_stdout(): + rootcol = py.test.collect.Directory(rootdir) + item = rootcol.getitembynames(funcprint_spec) + ex = BoxExecutor(item) + outcome_repr = ex.execute() + outcome = ReprOutcome(outcome_repr) + assert outcome.passed + assert outcome.stdout == "samfing\n" + +def test_box_executor_stdout_error(): + rootcol = py.test.collect.Directory(rootdir) + item = rootcol.getitembynames(funcprintfail_spec) + ex = BoxExecutor(item) + outcome_repr = ex.execute() + outcome = ReprOutcome(outcome_repr) + assert not outcome.passed + assert outcome.stdout == "samfing elz\n" Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Sat Aug 26 18:22:29 2006 @@ -7,7 +7,7 @@ from py.__.test.rsession.rsession import RSession from py.__.test.rsession.hostmanage import bin_rsync, init_hosts, teardown_hosts from py.__.test.rsession.testing.test_slave import (funcfail_spec, - funcpass_spec, funcskip_spec) + funcpass_spec, funcskip_spec, funcprint_spec, funcprintfail_spec) def setup_module(mod): mod.pkgdir = py.path.local(py.__file__).dirpath() @@ -171,12 +171,14 @@ itempass = rootcol.getitembynames(funcpass_spec) itemfail = rootcol.getitembynames(funcfail_spec) itemskip = rootcol.getitembynames(funcskip_spec) + itemprint = rootcol.getitembynames(funcprint_spec) # actually run some tests for node in nodes: node.send(itempass) node.send(itemfail) node.send(itemskip) + node.send(itemprint) teardown_hosts(allevents.append, [node.channel for node in nodes]) @@ -186,6 +188,9 @@ if i.outcome.passed] skipped = [i for i in events if i.outcome.skipped] - assert len(passed) == len(nodes) + assert len(passed) == 2 * len(nodes) assert len(skipped) == len(nodes) - assert len(events) == 3 * len(nodes) + assert len(events) == 4 * len(nodes) + # one of passed for each node has non-empty stdout + passed_stdout = [i for i in passed if i.outcome.stdout == 'samfing\n'] + assert len(passed_stdout) == len(nodes), passed Modified: py/branch/distributed/py/test/rsession/testing/test_slave.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_slave.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_slave.py Sat Aug 26 18:22:29 2006 @@ -18,9 +18,18 @@ def funcskip(): py.test.skip("skipped") +def funcprint(): + print "samfing" + +def funcprintfail(): + print "samfing elz" + asddsa + funcpass_spec = "py/test/rsession/testing/test_slave.py/funcpass".split("/") funcfail_spec = "py/test/rsession/testing/test_slave.py/funcfail".split("/") funcskip_spec = "py/test/rsession/testing/test_slave.py/funcskip".split("/") +funcprint_spec = "py/test/rsession/testing/test_slave.py/funcprint".split("/") +funcprintfail_spec = "py/test/rsession/testing/test_slave.py/funcprintfail".split("/") # ---------------------------------------------------------------------- From fijal at codespeak.net Sat Aug 26 18:30:35 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 Aug 2006 18:30:35 +0200 (CEST) Subject: [py-svn] r31688 - py/branch/distributed/py/test/rsession Message-ID: <20060826163035.E40391008F@code0.codespeak.net> Author: fijal Date: Sat Aug 26 18:30:34 2006 New Revision: 31688 Modified: py/branch/distributed/py/test/rsession/box.py Log: Flushing makes no sense here. Modified: py/branch/distributed/py/test/rsession/box.py ============================================================================== --- py/branch/distributed/py/test/rsession/box.py (original) +++ py/branch/distributed/py/test/rsession/box.py Sat Aug 26 18:30:34 2006 @@ -42,8 +42,6 @@ print str(i)[2:-1] print excinfo os._exit(1) - sys.stdout.flush() - sys.stderr.flush() os._exit(0) return pid From fijal at codespeak.net Sat Aug 26 18:58:02 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 Aug 2006 18:58:02 +0200 (CEST) Subject: [py-svn] r31693 - py/branch/distributed/py/test/rsession Message-ID: <20060826165802.0893A1009C@code0.codespeak.net> Author: fijal Date: Sat Aug 26 18:57:59 2006 New Revision: 31693 Modified: py/branch/distributed/py/test/rsession/conftest.py py/branch/distributed/py/test/rsession/hostmanage.py py/branch/distributed/py/test/rsession/report.py py/branch/distributed/py/test/rsession/rsession.py Log: Added waittime which tells how long server should wait for nodes to finish, otherwise report them as hanging. Modified: py/branch/distributed/py/test/rsession/conftest.py ============================================================================== --- py/branch/distributed/py/test/rsession/conftest.py (original) +++ py/branch/distributed/py/test/rsession/conftest.py Sat Aug 26 18:57:59 2006 @@ -4,5 +4,8 @@ option = py.test.Config.addoptions("distributed testing options", Option('-D', '', action="store", dest="disthosts", default=None, - help="comma separated list of testhosts", - )) + help="comma separated list of testhosts"), + Option('', '--waittime', + action="store", dest="waittime", default=10, + help="How long (in s) to wait for hanging nodes") + ) Modified: py/branch/distributed/py/test/rsession/hostmanage.py ============================================================================== --- py/branch/distributed/py/test/rsession/hostmanage.py (original) +++ py/branch/distributed/py/test/rsession/hostmanage.py Sat Aug 26 18:57:59 2006 @@ -5,6 +5,8 @@ setup_slave, MasterNode, dispatch_loop) from py.__.test.rsession import report +from py.__.test.rsession.conftest import option + def init_hosts(reporter, sshhosts, relpath, pkgdir): assert pkgdir.join("__init__.py").check(), ( "%s probably wrong" %(pkgdir,)) @@ -33,7 +35,12 @@ channel.send(None) for channel in channels: - report.wrapcall(reporter, channel.waitclose) + try: + report.wrapcall(reporter, channel.waitclose, int(option.waittime)) + except KeyboardInterrupt, SystemExit: + raise + except: + pass channel.gateway.exit() def bin_rsync(source, sshaddress, destpath): Modified: py/branch/distributed/py/test/rsession/report.py ============================================================================== --- py/branch/distributed/py/test/rsession/report.py (original) +++ py/branch/distributed/py/test/rsession/report.py Sat Aug 26 18:57:59 2006 @@ -82,5 +82,9 @@ def __init__(self): self.timeend = time.time() +class Nodes(ReportEvent): + def __init__(self, nodes): + self.nodes = nodes + # tryiter, main dispatch loop, something else (setup-teardown stuff) # Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Sat Aug 26 18:57:59 2006 @@ -61,8 +61,18 @@ self.timeend = item.timeend self.skips() self.failures() + self.hangs() self.summary() + def hangs(self): + h = [] + for node in self.nodes: + h += [(i, node.channel.gateway.sshaddress) for i in node.pending] + if h: + self.out.sep("=", " HANGING NODES ") + for i, node in h: + self.out.line("%s on %s" % (" ".join(i.listnames()), node)) + def failures(self): if self.failed_tests_outcome: self.out.sep("=", " FAILURES ") @@ -164,6 +174,9 @@ sshhost = event.channel.gateway.sshaddress itempath = " ".join(event.item.listnames()[1:]) print "%10s: %s %s" %(sshhost[:10], status, itempath) + + def report_Nodes(self, event): + self.nodes = event.nodes class RSession(object): """ @@ -197,6 +210,7 @@ # list(itemgenerator)]) dispatch_loop(nodes, itemgenerator, lambda : False) teardown_hosts(reporter, [node.channel for node in nodes]) + reporter(report.Nodes(nodes)) reporter(report.TestFinished()) def make_colitems(paths, baseon): From fijal at codespeak.net Sat Aug 26 19:44:35 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 Aug 2006 19:44:35 +0200 (CEST) Subject: [py-svn] r31694 - py/branch/distributed/py/test/rsession Message-ID: <20060826174435.85E84100A4@code0.codespeak.net> Author: fijal Date: Sat Aug 26 19:44:32 2006 New Revision: 31694 Modified: py/branch/distributed/py/test/rsession/box.py Log: (arigo, fijal) - fix flushing. Modified: py/branch/distributed/py/test/rsession/box.py ============================================================================== --- py/branch/distributed/py/test/rsession/box.py (original) +++ py/branch/distributed/py/test/rsession/box.py Sat Aug 26 19:44:32 2006 @@ -12,6 +12,8 @@ PYTESTSTDERR = "pyteststderr" PYTESTRETVAL = "pytestretval" +import tempfile + class Box(object): def __init__(self, fun, args = [], kwargs = {}): self.fun = fun @@ -19,16 +21,15 @@ self.kwargs = kwargs def run(self): - # What to do: - # XXX: Creates it in a current dir, and just open it, does not - # seem to be sane idea - try: - self.clear() - except OSError: - pass - os.mkfifo(PYTESTSTDOUT) - os.mkfifo(PYTESTSTDERR) - os.mkfifo(PYTESTRETVAL) + dirname = tempfile.mkdtemp() + #dirname = "." + self.dirname = dirname + self.PYTESTRETVAL = os.path.join(dirname, PYTESTRETVAL) + self.PYTESTSTDERR = os.path.join(dirname, PYTESTSTDERR) + self.PYTESTSTDOUT = os.path.join(dirname, PYTESTSTDOUT) + os.mkfifo(self.PYTESTSTDOUT) + os.mkfifo(self.PYTESTSTDERR) + os.mkfifo(self.PYTESTRETVAL) pid = os.fork() if pid: self.parent() @@ -49,28 +50,29 @@ # right now we need to call a function, but first we need to # map all IO that might happen # make sure sys.stdout points to file descriptor one - fdstdout = os.open(PYTESTSTDOUT, os.O_WRONLY) + fdstdout = os.open(self.PYTESTSTDOUT, os.O_WRONLY) if fdstdout != 1: os.dup2(fdstdout, 1) - fdstderr = os.open(PYTESTSTDERR, os.O_WRONLY) + fdstderr = os.open(self.PYTESTSTDERR, os.O_WRONLY) if fdstderr != 2: os.dup2(fdstderr, 2) - sys.stdout = os.fdopen(1, "w") - sys.stderr = os.fdopen(2, "w") - retvalf = open(PYTESTRETVAL, "w") + sys.stdout = os.fdopen(1, "w", 0) + sys.stderr = os.fdopen(2, "w", 0) + retvalf = open(self.PYTESTRETVAL, "w") retval = self.fun(*self.args, **self.kwargs) retvalf.write(marshal.dumps(retval)) retvalf.close() def parent(self): - self.fdstdout = open(PYTESTSTDOUT, "r") - self.fdstderr = open(PYTESTSTDERR, "r") - retval = open(PYTESTRETVAL, "r") + self.fdstdout = open(self.PYTESTSTDOUT, "r") + self.fdstderr = open(self.PYTESTSTDERR, "r") + retval = open(self.PYTESTRETVAL, "r") pid, exitstat = os.wait() self.signal = exitstat & 0x7f self.exitstat = exitstat & 0xff00 if not exitstat: retval_data = retval.read() + print retval_data self.retval = marshal.loads(retval_data) retval.close() else: @@ -84,8 +86,9 @@ def clear(self): try: - os.unlink(PYTESTSTDOUT) - os.unlink(PYTESTSTDERR) - os.unlink(PYTESTRETVAL) + os.unlink(self.PYTESTSTDOUT) + os.unlink(self.PYTESTSTDERR) + os.unlink(self.PYTESTRETVAL) + os.rmdir(self.dirname) except OSError: pass From fijal at codespeak.net Sat Aug 26 20:06:26 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 Aug 2006 20:06:26 +0200 (CEST) Subject: [py-svn] r31695 - py/branch/distributed/py/test/rsession Message-ID: <20060826180626.E377C100A4@code0.codespeak.net> Author: fijal Date: Sat Aug 26 20:06:25 2006 New Revision: 31695 Modified: py/branch/distributed/py/test/rsession/box.py py/branch/distributed/py/test/rsession/hostmanage.py Log: (arigo, fijal) - Fixed relative paths, so now it could probably work on NFS. Modified: py/branch/distributed/py/test/rsession/box.py ============================================================================== --- py/branch/distributed/py/test/rsession/box.py (original) +++ py/branch/distributed/py/test/rsession/box.py Sat Aug 26 20:06:25 2006 @@ -22,7 +22,6 @@ def run(self): dirname = tempfile.mkdtemp() - #dirname = "." self.dirname = dirname self.PYTESTRETVAL = os.path.join(dirname, PYTESTRETVAL) self.PYTESTSTDERR = os.path.join(dirname, PYTESTSTDERR) Modified: py/branch/distributed/py/test/rsession/hostmanage.py ============================================================================== --- py/branch/distributed/py/test/rsession/hostmanage.py (original) +++ py/branch/distributed/py/test/rsession/hostmanage.py Sat Aug 26 20:06:25 2006 @@ -18,10 +18,13 @@ exc_info = [None] def host_init(host): + # XXX: because of NFS we do create different directories + # otherwise, .pyc files overlap + real_relpath = relpath + "-" + host report.wrapcall(reporter, bin_rsync, - str(pkgdir.dirpath())+"/", host, relpath) + str(pkgdir.dirpath())+"/", host, real_relpath) gw = py.execnet.SshGateway(host) - ch = setup_slave(gw, os.path.join(relpath, pkgdir.basename)) + ch = setup_slave(gw, os.path.join(real_relpath, pkgdir.basename)) nodes.append(MasterNode(ch, reporter)) reporter(report.HostReady(ch.gateway.sshaddress)) From fijal at codespeak.net Sun Aug 27 12:51:46 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 27 Aug 2006 12:51:46 +0200 (CEST) Subject: [py-svn] r31713 - in py/branch/distributed/py/test: . rsession rsession/testing testing Message-ID: <20060827105146.CE293100CC@code0.codespeak.net> Author: fijal Date: Sun Aug 27 12:51:43 2006 New Revision: 31713 Modified: py/branch/distributed/py/test/collect.py py/branch/distributed/py/test/rsession/report.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_rsession.py py/branch/distributed/py/test/testing/test_collect.py Log: Fix some bugs, added select-by-keyword command line argument. Modified: py/branch/distributed/py/test/collect.py ============================================================================== --- py/branch/distributed/py/test/collect.py (original) +++ py/branch/distributed/py/test/collect.py Sun Aug 27 12:51:43 2006 @@ -176,28 +176,52 @@ newl.append(x.name) return ".".join(newl) - def tryiter(self, yieldtype=None, reporterror=None): + # XXX: Copied from session + def skipbykeyword(self, keyword): + if not keyword: + return + chain = self.listchain() + for key in filter(None, keyword.split()): + eor = key[:1] == '-' + if eor: + key = key[1:] + if not (eor ^ self._matchonekeyword(key, chain)): + py.test.skip("test not selected by keyword %r" %(keyword,)) + + def _matchonekeyword(self, key, chain): + for subitem in chain: + if subitem.haskeyword(key): + return True + return False + + def tryiter(self, yieldtype=None, reporterror=None, keyword=None): """ yield stop item instances from flattening the collector. XXX deprecated: this way of iteration is not safe in all - cases. + cases. Mostly fixed, need to introduce skipped-by-keyword """ if yieldtype is None: yieldtype = py.test.Item - if isinstance(self, yieldtype): - yield self + if isinstance(self, yieldtype): + try: + self.skipbykeyword(keyword) + yield self + except py.test.Item.Skipped: + if reporterror is not None: + excinfo = py.code.ExceptionInfo() + reporterror((excinfo, self)) else: if not isinstance(self, py.test.Item): try: for x in self.run(): - for y in self.join(x).tryiter(yieldtype): + for y in self.join(x).tryiter(yieldtype, reporterror, keyword): yield y except KeyboardInterrupt: raise except: if reporterror is not None: excinfo = py.code.ExceptionInfo() - reporterror(excinfo) + reporterror((excinfo, self)) def _prepare(self): if not hasattr(self, '_name2items'): Modified: py/branch/distributed/py/test/rsession/report.py ============================================================================== --- py/branch/distributed/py/test/rsession/report.py (original) +++ py/branch/distributed/py/test/rsession/report.py Sun Aug 27 12:51:43 2006 @@ -86,5 +86,15 @@ def __init__(self, nodes): self.nodes = nodes +class SkippedTryiter(ReportEvent): + def __init__(self, excinfo, item): + self.excinfo = excinfo + self.item = item + +class FailedTryiter(ReportEvent): + def __init__(self, excinfo, item): + self.excinfo = excinfo + self.item = item + # tryiter, main dispatch loop, something else (setup-teardown stuff) # Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Sun Aug 27 12:51:43 2006 @@ -119,15 +119,19 @@ texts = {} for event in self.skipped_tests_outcome: colitem = event.item - outcome = event.outcome - text = outcome.skipped - itemname = event.channel.gateway.sshaddress + ":" + \ - "/".join(colitem.listnames()) + if isinstance(event, report.ReceivedItemOutcome): + outcome = event.outcome + text = outcome.skipped + itemname = event.channel.gateway.sshaddress + ":" + \ + "/".join(colitem.listnames()) + elif isinstance(event, report.SkippedTryiter): + text = str(event.excinfo.value) + itemname = "/".join(colitem.listnames()) if text not in texts: texts[text] = [itemname] else: texts[text].append(itemname) - + if texts: self.out.line() self.out.sep('_', 'reasons for skipped tests') @@ -157,6 +161,14 @@ self.out.sep("=", " %d test run%s%s in %.2fs " % (total, skipped_str, failed_str, self.timeend - self.timestart)) + def report_SkippedTryiter(self, event): + #event.outcome.excinfo.source = + self.skipped_tests_outcome.append(event) + + def report_FailedTryiter(self, event): + pass + # XXX: right now we do not do anything with it + def report_ReceivedItemOutcome(self, event): host = event.channel.gateway.sshaddress if event.outcome.passed: @@ -199,10 +211,19 @@ destrelpath = "pytestcache" nodes = init_hosts(reporter, sshhosts, destrelpath, pkgdir) + + def reporterror(data): + excinfo, item = data + if excinfo.type is py.test.Item.Skipped: + reporter(report.SkippedTryiter(excinfo, item)) + else: + reporter(report.FailedTryiter(excinfo, item)) + + keyword = self.config.option.keyword def itemgen(): for x in colitems: - for y in x.tryiter(): + for y in x.tryiter(reporterror = reporterror, keyword = keyword): yield y itemgenerator = itemgen() Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Sun Aug 27 12:51:43 2006 @@ -50,6 +50,23 @@ assert col_two.listnames() == [pkgdir.dirpath().basename, "py", "path", "__init__.py"] +def test_example_tryiter(): + events = [] + tmpdir = py.test.ensuretemp("tryitertest") + tmpdir.ensure("a", "__init__.py") + tmpdir.ensure("conftest.py").write(py.code.Source(""" + import py + py.test.skip("Reason") + """)) + tmpdir.ensure("a", "test_empty.py").write(py.code.Source(""" + def test_empty(): + pass + """)) + rootcol = py.test.collect.Directory(tmpdir) + data = list(rootcol.tryiter(reporterror=events.append)) + assert len(events) == 1 + assert str(events[0][0].value) == "Reason" + class TestWithRealSshHosts: def setup_class(cls): from py.__.test.rsession.conftest import option @@ -89,7 +106,7 @@ channel.send(None) res = channel.receive() assert res == "ok" - + def test_example_distribution(self): # XXX find a better way for the below tmpdir = py.path.local(py.__file__).dirpath().dirpath() Modified: py/branch/distributed/py/test/testing/test_collect.py ============================================================================== --- py/branch/distributed/py/test/testing/test_collect.py (original) +++ py/branch/distributed/py/test/testing/test_collect.py Sun Aug 27 12:51:43 2006 @@ -366,7 +366,8 @@ l = [] list(col.tryiter(reporterror=l.append)) assert len(l) == 1 - assert isinstance(l[0], py.code.ExceptionInfo) + excinfo, item = l[0] + assert isinstance(excinfo, py.code.ExceptionInfo) def test_tryiter_handles_keyboardinterrupt(): tmp = py.test.ensuretemp("tryiterkeyboard") From fijal at codespeak.net Sun Aug 27 15:38:27 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 27 Aug 2006 15:38:27 +0200 (CEST) Subject: [py-svn] r31730 - py/branch/distributed Message-ID: <20060827133827.97451100B4@code0.codespeak.net> Author: fijal Date: Sun Aug 27 15:38:25 2006 New Revision: 31730 Modified: py/branch/distributed/TODO Log: Added DONE stuff. Modified: py/branch/distributed/TODO ============================================================================== --- py/branch/distributed/TODO (original) +++ py/branch/distributed/TODO Sun Aug 27 15:38:25 2006 @@ -14,20 +14,20 @@ open issues: * sending cmdline and other options to remote places -* capturing stdout/stderr remotely (and sending it back) -* noticing hanging tests/hosts +DONE * capturing stdout/stderr remotely (and sending it back) +DONE * noticing hanging tests/hosts * reacting to hanging tests/hosts (probably first just remove the masternode from the dispatching list and reschedule pending tests to a different host and report about all this) * more events, at least: - - start testing - - finish testing - - setup/teardown of hosts - - segfaults + DONE - start testing + DONE - finish testing + DONE - setup/teardown of hosts + DONE - segfaults - rsyncing (after 1-n rsync) * nice hostmanagement * 1-n rsync -* improved reporting, e.g. +WIP * improved reporting, e.g. report which packages are used from where on the remote side * lazy remote py lib import? * --pdb -> remote screen sessions From fijal at codespeak.net Sun Aug 27 17:21:47 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 27 Aug 2006 17:21:47 +0200 (CEST) Subject: [py-svn] r31740 - in py/branch/distributed/py/test: . rsession rsession/testing rsession/webdata Message-ID: <20060827152147.BF19B100F3@code0.codespeak.net> Author: fijal Date: Sun Aug 27 17:21:42 2006 New Revision: 31740 Added: py/branch/distributed/py/test/rsession/web.py (contents, props changed) py/branch/distributed/py/test/rsession/webdata/ py/branch/distributed/py/test/rsession/webdata/index.html (contents, props changed) py/branch/distributed/py/test/rsession/webjs.py (contents, props changed) Modified: py/branch/distributed/py/test/collect.py py/branch/distributed/py/test/rsession/conftest.py py/branch/distributed/py/test/rsession/report.py py/branch/distributed/py/test/rsession/rsession.py py/branch/distributed/py/test/rsession/testing/test_rsession.py Log: Added simple web toolkit for displaying tests. This is immediate checkin for someone to play eventually around, I think I'll come back to that later. Modified: py/branch/distributed/py/test/collect.py ============================================================================== --- py/branch/distributed/py/test/collect.py (original) +++ py/branch/distributed/py/test/collect.py Sun Aug 27 17:21:42 2006 @@ -213,6 +213,8 @@ else: if not isinstance(self, py.test.Item): try: + if reporterror is not None: + reporterror((None, self)) for x in self.run(): for y in self.join(x).tryiter(yieldtype, reporterror, keyword): yield y Modified: py/branch/distributed/py/test/rsession/conftest.py ============================================================================== --- py/branch/distributed/py/test/rsession/conftest.py (original) +++ py/branch/distributed/py/test/rsession/conftest.py Sun Aug 27 17:21:42 2006 @@ -6,6 +6,6 @@ action="store", dest="disthosts", default=None, help="comma separated list of testhosts"), Option('', '--waittime', - action="store", dest="waittime", default=10, + action="store", dest="waittime", default=100, help="How long (in s) to wait for hanging nodes") ) Modified: py/branch/distributed/py/test/rsession/report.py ============================================================================== --- py/branch/distributed/py/test/rsession/report.py (original) +++ py/branch/distributed/py/test/rsession/report.py Sun Aug 27 17:21:42 2006 @@ -96,5 +96,9 @@ self.excinfo = excinfo self.item = item -# tryiter, main dispatch loop, something else (setup-teardown stuff) -# +class ItemStart(ReportEvent): + """ This class shows most of the start stuff, like directory, module, class + can be used for containers + """ + def __init__(self, item): + self.item = item Modified: py/branch/distributed/py/test/rsession/rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/rsession.py (original) +++ py/branch/distributed/py/test/rsession/rsession.py Sun Aug 27 17:21:42 2006 @@ -214,7 +214,9 @@ def reporterror(data): excinfo, item = data - if excinfo.type is py.test.Item.Skipped: + if excinfo is None: + reporter(report.ItemStart(item)) + elif excinfo.type is py.test.Item.Skipped: reporter(report.SkippedTryiter(excinfo, item)) else: reporter(report.FailedTryiter(excinfo, item)) Modified: py/branch/distributed/py/test/rsession/testing/test_rsession.py ============================================================================== --- py/branch/distributed/py/test/rsession/testing/test_rsession.py (original) +++ py/branch/distributed/py/test/rsession/testing/test_rsession.py Sun Aug 27 17:21:42 2006 @@ -64,8 +64,8 @@ """)) rootcol = py.test.collect.Directory(tmpdir) data = list(rootcol.tryiter(reporterror=events.append)) - assert len(events) == 1 - assert str(events[0][0].value) == "Reason" + assert len(events) == 2 + assert str(events[1][0].value) == "Reason" class TestWithRealSshHosts: def setup_class(cls): Added: py/branch/distributed/py/test/rsession/web.py ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/web.py Sun Aug 27 17:21:42 2006 @@ -0,0 +1,120 @@ + +""" web server for py.test +""" + +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler + + +from _findpy import py +import thread, threading + + +from pypy.translator.js import commproxy + +commproxy.USE_MOCHIKIT = False + +import sys +from pypy.rpython.ootypesystem.bltregistry import MethodDesc, BasicExternal, \ + described +from py.__.test.rsession.rsession import RSession +from py.__.test.rsession import report +from pypy.translator.js.main import rpython2javascript + +DATADIR = "webdata" + +def escape(s): + return s.replace("&", "&").replace("<", "<").replace(">", ">"). \ + replace("'", "\\'").replace(" ", " ") + + +class ExportedMethods(BasicExternal): + _render_xmlhttp = True + def __init__(self): + self.wait_flag = threading.Condition() + self.pending_events = [] + + @described(retval={"aa":"aa"}) + def show_status_change(self): + while 1: + try: + self.wait_flag.acquire() + self.wait_flag.wait(1) + if self.pending_events: + event = self.pending_events.pop() + # some dispatcher here + if isinstance(event, report.ReceivedItemOutcome): + args = ["%s: '%s'" % (key, escape(str(value))) \ + for key, value in event.outcome.__dict__.iteritems()] + elif isinstance(event, report.ItemStart): + item = event.item + item_tuple = "itemtype: '%s', itemname: '%s'" % (\ + item.__class__.__name__, item.name) + args = [item_tuple] + else: + args = [] + args.append("event: '%s'" % escape(str(event))) + + type_tuple = "type: '%s'" % event.__class__.__name__ + return "{%s}" % ",".join(args + [type_tuple]) + finally: + self.wait_flag.release() + + def report(self, event): + try: + self.wait_flag.acquire() + self.pending_events.insert(0, event) + self.wait_flag.notify() + finally: + self.wait_flag.release() + +exported_methods = ExportedMethods() + +class TestHandler(BaseHTTPRequestHandler): + def do_GET(self): + path = self.path + if path.endswith("/"): + path = path[:-1] + if path.startswith("/"): + path = path[1:] + method_to_call = getattr(self, "run_" + path, None) + if method_to_call is None: + exec_meth = getattr(exported_methods, path, None) + if exec_meth is None: + self.send_error(404, "File %s not found" % path) + else: + self.serve_data('text/json', exec_meth()) + else: + method_to_call() + + do_POST = do_GET + + def run_(self): + self.run_index() + + def run_index(self): + data = py.path.local(DATADIR).join("index.html").open().read() + self.serve_data("text/html", data) + + def run_jssource(self): + from py.__.test.rsession import webjs + self.serve_data("text/javascript", rpython2javascript(webjs, ["main"])) + + def serve_data(self, content_type, data): + self.send_response(200) + self.send_header("Content-type", content_type) + self.send_header("Content-length", len(data)) + self.end_headers() + self.wfile.write(data) + + +def _main(args): + server_address = ('', 8000) + # do some options parsing + config, args = py.test.Config.parse(args) + thread.start_new_thread(RSession(config).main, (args, \ + exported_methods.report)) + httpd = HTTPServer(server_address, TestHandler) + httpd.serve_forever() + +if __name__ == '__main__': + _main(sys.argv) Added: py/branch/distributed/py/test/rsession/webdata/index.html ============================================================================== --- (empty file) +++ py/branch/distributed/py/test/rsession/webdata/index.html Sun Aug 27 17:21:42 2006 @@ -0,0 +1,12 @@ + + + Py.test +