[pypy-svn] r35597 - in pypy/dist/pypy/tool/build: . bin test
guido at codespeak.net
guido at codespeak.net
Mon Dec 11 20:39:29 CET 2006
Author: guido
Date: Mon Dec 11 20:39:17 2006
New Revision: 35597
Modified:
pypy/dist/pypy/tool/build/bin/client
pypy/dist/pypy/tool/build/bin/startcompile
pypy/dist/pypy/tool/build/client.py
pypy/dist/pypy/tool/build/config.py
pypy/dist/pypy/tool/build/server.py
pypy/dist/pypy/tool/build/test/test_client.py
pypy/dist/pypy/tool/build/test/test_pypybuilder.py
pypy/dist/pypy/tool/build/test/test_server.py
Log:
Capturing stdout/stderr and sending to the server when done, cleaning up build
paths that aren't 'done' (no log written to it) on startup.
Modified: pypy/dist/pypy/tool/build/bin/client
==============================================================================
--- pypy/dist/pypy/tool/build/bin/client (original)
+++ pypy/dist/pypy/tool/build/bin/client Mon Dec 11 20:39:17 2006
@@ -6,7 +6,7 @@
from pypy.tool.build import config as buildconfig
from py.execnet import SshGateway, PopenGateway
-from pypy.tool.build.client import init, zip_result
+from pypy.tool.build.client import init, zip_result, OutputBuffer
from pypy.config.config import to_optparse, Config
from pypy.config import pypyoption
@@ -52,14 +52,24 @@
config = pypyoption.get_pypy_config()
config.override(compileinfo)
- # XXX compile here...
- driver = TranslationDriver.from_targetspec(
- targetpypystandalone.__dict__, config=config,
- default_goal='compile')
- driver.proceed(['compile'])
+ buffer = OutputBuffer(sys.__stderr__)
+ sys.stdout = buffer
+ sys.stderr = buffer
+ try:
+ driver = TranslationDriver.from_targetspec(
+ targetpypystandalone.__dict__, config=config,
+ default_goal='compile')
+ driver.proceed(['compile'])
+ finally:
+ sys.stdout = sys.__stdout__
+ sys.stderr = sys.__stderr__
+ # send over zip data, end with a None
zip_result(udir, channel)
+ # send over logs
+ channel.send(buffer.getvalue())
+
print 'done with compilation, waiting for next'
except EOFError:
sys.exit()
Modified: pypy/dist/pypy/tool/build/bin/startcompile
==============================================================================
--- pypy/dist/pypy/tool/build/bin/startcompile (original)
+++ pypy/dist/pypy/tool/build/bin/startcompile Mon Dec 11 20:39:17 2006
@@ -41,8 +41,8 @@
from pypy.tool.build import execnetconference
conference = execnetconference.conference(gw, port, False)
- channel = conference.remote_exec(initcode % (path, email,
- sysinfo, compileinfo))
+ channel = conference.remote_exec(initcode % (path, email, sysinfo,
+ compileinfo))
return channel
if __name__ == '__main__':
@@ -66,8 +66,8 @@
gw = PopenGateway()
else:
gw = SshGateway(config.server)
- channel = init(gw, sysinfo, compileinfo, config.path, args[0],
- port=config.port)
+ channel = init(gw, sysinfo, compileinfo, config.path, args[0],
+ port=config.port)
data = channel.receive()
if type(data) == str:
print data
@@ -79,7 +79,7 @@
ispath, data = data
if ispath:
print ('a suitable result is already available, you can find it '
- 'at "%s"' % (data,))
+ 'at "%s"' % (data,))
else:
print data
print 'you will be mailed once it\'s ready'
Modified: pypy/dist/pypy/tool/build/client.py
==============================================================================
--- pypy/dist/pypy/tool/build/client.py (original)
+++ pypy/dist/pypy/tool/build/client.py Mon Dec 11 20:39:17 2006
@@ -1,7 +1,7 @@
import thread
import py
from zipfile import ZipFile
-
+from cStringIO import StringIO
class PPBClient(object):
def __init__(self, channel, sysinfo, testing=False):
@@ -22,10 +22,10 @@
def compile(self, info):
"""send a compile job to the client side"""
- self.busy_on = info
self.channel.send(info)
accepted = self.channel.receive()
if accepted:
+ self.busy_on = info
thread.start_new_thread(self.wait_until_done, (info,))
else:
self.refused.append(info)
@@ -44,12 +44,15 @@
except EOFError:
# stop compilation, client has disconnected
return
+ # end of data is marked by sending a None
if chunk is None:
break
fp.write(chunk)
finally:
fp.close()
-
+ # write the log (process stdout/stderr) to the buildpath
+ buildpath.log = self.channel.receive()
+
self.server.compilation_done(info, buildpath)
self.busy_on = None
@@ -123,3 +126,17 @@
print exc
continue
zip.close()
+
+class OutputBuffer(object):
+ def __init__(self, print_channel=None):
+ self.print_channel = print_channel
+ self.buffer = StringIO()
+
+ def write(self, s):
+ self.buffer.write(s)
+ if self.print_channel:
+ self.print_channel.write(s)
+
+ def getvalue(self):
+ return self.buffer.getvalue()
+
Modified: pypy/dist/pypy/tool/build/config.py
==============================================================================
--- pypy/dist/pypy/tool/build/config.py (original)
+++ pypy/dist/pypy/tool/build/config.py Mon Dec 11 20:39:17 2006
@@ -5,6 +5,7 @@
# general settings, used by both server and client
server = 'localhost'
port = 12321
+testport = 32123
path = [str(packageparent)]
# configuration of options for client and startcompile
Modified: pypy/dist/pypy/tool/build/server.py
==============================================================================
--- pypy/dist/pypy/tool/build/server.py (original)
+++ pypy/dist/pypy/tool/build/server.py Mon Dec 11 20:39:17 2006
@@ -132,6 +132,21 @@
zipfile = property(_zipfile, _set_zipfile)
+ def _log(self):
+ log = self.join('log')
+ if not log.check():
+ return ''
+ return log.read()
+
+ def _set_log(self, data):
+ self.join('log').write(data)
+
+ log = property(_log, _set_log)
+
+ def _done(self):
+ return not not self.log
+ done = property(_done)
+
class PPBServer(object):
retry_interval = 10
@@ -146,8 +161,13 @@
self._buildpath = py.path.local(builddir)
self._clients = []
- info_to_path = [(p.info, str(p)) for p in
- self._get_buildpaths(builddir)]
+ info_to_path = []
+ for bp in self._get_buildpaths(builddir):
+ if bp.done:
+ info_to_path.append((bp.info, str(bp)))
+ else:
+ # throw away half-done builds...
+ bp.remove()
self._requeststorage = RequestStorage(info_to_path)
self._queued = []
Modified: pypy/dist/pypy/tool/build/test/test_client.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_client.py (original)
+++ pypy/dist/pypy/tool/build/test/test_client.py Mon Dec 11 20:39:17 2006
@@ -2,6 +2,7 @@
from pypy.tool.build import client
import py
import time
+import sys
from fake import FakeChannel, FakeServer
class ClientForTests(client.PPBClient):
@@ -31,8 +32,10 @@
assert accepted
ret = c1.channel.receive()
assert ret == info # this was still in the buffer
+ assert c1.busy_on == info
c1.channel.send('foo bar')
c1.channel.send(None)
+ c1.channel.send('log')
# meanwhile the client starts a thread that waits until there's data
# available on its own channel, with our FakeChannel it has data rightaway,
@@ -44,6 +47,7 @@
assert done[0] == info
assert done[1] == (temp / 'build-0')
+ assert temp.join('build-0/log').read() == 'log'
def test_channelwrapper():
class FakeChannel(object):
@@ -67,4 +71,22 @@
accepted = c1.compile(info)
assert not accepted
assert info in c1.refused
+ assert c1.busy_on == None
+
+def test_output_buffer():
+ b = client.OutputBuffer()
+ sys.stdout = b
+ try:
+ print 'foo'
+ finally:
+ sys.stdout = sys.__stdout__
+ assert b.getvalue() == 'foo\n'
+ s = py.std.StringIO.StringIO()
+ b = client.OutputBuffer(s)
+ sys.stdout = b
+ try:
+ print 'bar'
+ finally:
+ sys.stdout = sys.__stdout__
+ assert b.getvalue() == s.getvalue() == 'bar\n'
Modified: pypy/dist/pypy/tool/build/test/test_pypybuilder.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_pypybuilder.py (original)
+++ pypy/dist/pypy/tool/build/test/test_pypybuilder.py Mon Dec 11 20:39:17 2006
@@ -28,7 +28,7 @@
# first initialize a server
sgw = py.execnet.PopenGateway()
temppath = py.test.ensuretemp('pypybuilder-functional')
- sc = server.init(sgw, port=config.port, path=config.testpath,
+ sc = server.init(sgw, port=config.testport, path=config.testpath,
buildpath=str(temppath))
# give the server some time to wake up
@@ -37,12 +37,14 @@
# then two clients, both with different system info
sysconfig1 = _get_sysconfig()
cgw1 = py.execnet.PopenGateway()
- cc1 = client.init(cgw1, sysconfig1, port=config.port, testing=True)
+ cc1 = client.init(cgw1, sysconfig1, port=config.testport, testing=True)
+ cc1.receive() # welcome message
sysconfig2 = _get_sysconfig()
sysconfig2.foo = 2
cgw2 = py.execnet.PopenGateway()
- cc2 = client.init(cgw2, sysconfig2, port=config.port, testing=True)
+ cc2 = client.init(cgw2, sysconfig2, port=config.testport, testing=True)
+ cc2.receive() # welcome message
# give the clients some time to register themselves
py.std.time.sleep(SLEEP_INTERVAL)
@@ -57,14 +59,11 @@
channel.close()
"""
compgw = py.execnet.PopenGateway()
- compconf = execnetconference.conference(compgw, config.port)
-
- # clients normally respond with a boolean on a compilation request... to
- # allow the code to continue, already put some Trues (meaning 'I accept
- # the job') in the buffers
- cc1.send(True)
- cc2.send(True)
+ compconf = execnetconference.conference(compgw, config.testport)
+ # we're going to have to closely mimic the bin/client script to avoid
+ # freezes (from the app waiting for input)
+
# this one should fail because there's no client found for foo = 3
compc = compconf.remote_exec(code % (config.testpath, 'foo1 at bar.com',
{'foo': 3}))
@@ -80,10 +79,14 @@
compc = compconf.remote_exec(code % (config.testpath, 'foo2 at bar.com',
{'foo': 1}))
+ # client 1 will now send a True to the server to tell it wants to compile
+ cc1.send(True)
+
# and another one
py.std.time.sleep(SLEEP_INTERVAL)
ret = compc.receive()
+ print repr(ret)
assert not ret[0]
assert ret[1].find('found a suitable client') > -1
@@ -91,7 +94,6 @@
py.std.time.sleep(SLEEP_INTERVAL)
# client 1 should by now have received the info to build for
- cc1.receive() # 'welcome'
ret = cc1.receive()
assert ret == ({'foo': 1}, {})
@@ -102,7 +104,7 @@
sysconfig3 = _get_sysconfig()
sysconfig3.foo = 3
cgw3 = py.execnet.PopenGateway()
- cc3 = client.init(cgw3, sysconfig3, port=config.port, testing=True)
+ cc3 = client.init(cgw3, sysconfig3, port=config.testport, testing=True)
# add True to the buffer just like we did for channels 1 and 2
cc3.send(True)
@@ -124,9 +126,10 @@
channel.close()
"""
compgw2 = py.execnet.PopenGateway()
- compconf2 = execnetconference.conference(compgw2, config.port)
+ compconf2 = execnetconference.conference(compgw2, config.testport)
compc2 = compconf2.remote_exec(code % (config.testpath, SLEEP_INTERVAL))
+ cc2.send(True)
# we check whether all emails are now sent, since after adding the third
# client, and calling _try_queued(), both jobs should have been processed
Modified: pypy/dist/pypy/tool/build/test/test_server.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_server.py (original)
+++ pypy/dist/pypy/tool/build/test/test_server.py Mon Dec 11 20:39:17 2006
@@ -94,13 +94,13 @@
def test_buildpath():
tempdir = py.test.ensuretemp('pypybuilder-buildpath')
# grmbl... local.__new__ checks for class equality :(
- bp = BuildPath(str(tempdir / 'test1'))
+ bp = BuildPath(str(tempdir / 'test1'))
assert not bp.check()
assert bp.info == ({}, {})
bp.info = ({'foo': 1, 'bar': [1,2]}, {'baz': 1})
assert bp.info == ({'foo': 1, 'bar': [1,2]}, {'baz': 1})
- assert (sorted((bp / 'system_info.txt').readlines()) ==
+ assert (sorted((bp / 'system_info.txt').readlines()) ==
['bar: [1, 2]\n', 'foo: 1\n'])
assert isinstance(bp.zipfile, py.path.local)
@@ -138,3 +138,15 @@
path2.remove()
finally:
path1.remove()
+
+def test_cleanup_old_builds():
+ temppath = py.test.ensuretemp('cleanup_old_builds')
+ bp1 = server.BuildPath(temppath.join('bp1'))
+ bp1.ensure(dir=True)
+ bp2 = server.BuildPath(temppath.join('bp2'))
+ bp2.ensure(dir=True)
+ bp2.log = 'log'
+ svr = server.PPBServer('test', FakeChannel(), str(temppath))
+ assert not bp1.check()
+ assert bp2.check()
+
More information about the Pypy-commit
mailing list