[pypy-svn] r35906 - in pypy/dist/pypy/tool/build: . bin test
guido at codespeak.net
guido at codespeak.net
Tue Dec 19 22:52:29 CET 2006
Author: guido
Date: Tue Dec 19 22:52:26 2006
New Revision: 35906
Modified:
pypy/dist/pypy/tool/build/bin/client
pypy/dist/pypy/tool/build/bin/server
pypy/dist/pypy/tool/build/client.py
pypy/dist/pypy/tool/build/conftest.py
pypy/dist/pypy/tool/build/server.py
pypy/dist/pypy/tool/build/test/test_pypybuilder.py
pypy/dist/pypy/tool/build/test/test_server.py
Log:
Lots of small cleanups, added some log lines, fixed problems with the
additional test option (--functional), using py.code.Source instead of raw
strings where appropriate, removed 'welcome' message. Also refactored the
functional tests, they're a lot cleaner now.
Modified: pypy/dist/pypy/tool/build/bin/client
==============================================================================
--- pypy/dist/pypy/tool/build/bin/client (original)
+++ pypy/dist/pypy/tool/build/bin/client Tue Dec 19 22:52:26 2006
@@ -2,69 +2,65 @@
import py
from py.execnet import PopenGateway
+from pypy.tool.build import outputbuffer
def compile(wc, compileinfo):
- code = """\
-import sys
-import os
-import traceback
-
-# interpolating the path
-pypath = %r
-
-sys.path = [pypath] + sys.path
-os.chdir(pypath)
-
-# nasty, interpolating a large chunk of code (OutputBuffer)
-%s
-
-# interpolating config
-compileinfo = %r
-
-# log locally too
-log = open('/tmp/buildclient.log', 'a')
-outbuffer = OutputBuffer(log)
-sys.stdout = outbuffer
-sys.stderr = outbuffer
-try:
- try:
- from pypy.interpreter.error import OperationError
- from pypy.translator.goal import targetpypystandalone
- from pypy.translator.driver import TranslationDriver
- from pypy.config import pypyoption
- from pypy.tool.udir import udir
-
- config = pypyoption.get_pypy_config()
- config.override(compileinfo)
-
- driver = TranslationDriver.from_targetspec(
- targetpypystandalone.__dict__, config=config,
- default_goal='compile')
- driver.proceed(['compile'])
- except Exception, e:
- # XXX we may want to check
- exception_occurred = True
- exc, e, tb = sys.exc_info()
- print '=' * 79
- print 'Exception during compilation:'
- print '%%s: %%s' %% (exc, e)
- print
- print '\\n'.join(traceback.format_tb(tb))
- print '=' * 79
- del tb
- channel.send(None)
- else:
- channel.send(str(udir))
-finally:
- sys.stdout = sys.__stdout__
- sys.stderr = sys.__stderr__
- log.close()
-channel.send(outbuffer.getvalue())
-channel.close()
-"""
+ code = py.code.Source(outputbuffer, """\
+ import sys
+ import os
+ import traceback
+
+ # interpolating the path
+ pypath = %r
+
+ sys.path = [pypath] + sys.path
+ os.chdir(pypath)
+
+ # interpolating config
+ compileinfo = %r
+
+ # log locally too
+ log = open('/tmp/buildclient.log', 'a')
+ outbuffer = OutputBuffer(log)
+ sys.stdout = outbuffer
+ sys.stderr = outbuffer
+ try:
+ try:
+ from pypy.interpreter.error import OperationError
+ from pypy.translator.goal import targetpypystandalone
+ from pypy.translator.driver import TranslationDriver
+ from pypy.config import pypyoption
+ from pypy.tool.udir import udir
+
+ config = pypyoption.get_pypy_config()
+ config.override(compileinfo)
+
+ driver = TranslationDriver.from_targetspec(
+ targetpypystandalone.__dict__, config=config,
+ default_goal='compile')
+ driver.proceed(['compile'])
+ except Exception, e:
+ # XXX we may want to check
+ exception_occurred = True
+ exc, e, tb = sys.exc_info()
+ print '=' * 79
+ print 'Exception during compilation:'
+ print '%%s: %%s' %% (exc, e)
+ print
+ print '\\n'.join(traceback.format_tb(tb))
+ print '=' * 79
+ del tb
+ channel.send(None)
+ else:
+ channel.send(str(udir))
+ finally:
+ sys.stdout = sys.__stdout__
+ sys.stderr = sys.__stderr__
+ log.close()
+ channel.send(outbuffer.getvalue())
+ channel.close()
+ """)
gw = PopenGateway()
- buffercode = py.magic.autopath().dirpath().dirpath()\
- .join('outputbuffer.py').read()
interpolated = code % (str(wc), buffercode, compileinfo)
channel = gw.remote_exec(interpolated)
try:
Modified: pypy/dist/pypy/tool/build/bin/server
==============================================================================
--- pypy/dist/pypy/tool/build/bin/server (original)
+++ pypy/dist/pypy/tool/build/bin/server Tue Dec 19 22:52:26 2006
@@ -13,7 +13,7 @@
gw = PopenGateway()
else:
gw = SshGateway(config.server)
- channel = init(gw, port=config.port, path=config.path,
+ channel = init(gw, port=config.port, path=config.path,
projectname=config.projectname,
buildpath=str(config.buildpath),
mailhost=config.mailhost,
@@ -29,3 +29,4 @@
finally:
channel.close()
gw.exit()
+
Modified: pypy/dist/pypy/tool/build/client.py
==============================================================================
--- pypy/dist/pypy/tool/build/client.py (original)
+++ pypy/dist/pypy/tool/build/client.py Tue Dec 19 22:52:26 2006
@@ -5,12 +5,12 @@
from pypy.tool.build import build
class PPBClient(object):
- def __init__(self, channel, sysinfo, testing=False):
+ def __init__(self, channel, sysinfo, testing_sleeptime=False):
self.channel = channel
self.sysinfo = sysinfo
self.busy_on = None
self.refused = []
- self.testing = testing
+ self.testing_sleeptime = testing_sleeptime
from pypy.tool.build import ppbserver
self.server = ppbserver
@@ -35,7 +35,7 @@
def wait_until_done(self, request):
buildpath = self.server.get_new_buildpath(request)
- if not self.testing:
+ if not self.testing_sleeptime:
fp = buildpath.zipfile.open('w')
gotdata = False
try:
@@ -55,6 +55,11 @@
fp.close()
# write the log (process stdout/stderr) to the buildpath
buildpath.log = self.channel.receive()
+ else:
+ # pretend we're compiling by sleeping a bit...
+ open('/tmp/test_client_functional_crap', 'a').write('CLIENT - starting compile, sleeping %r secs\n' % (self.testing_sleeptime,))
+ py.std.time.sleep(self.testing_sleeptime)
+ open('/tmp/test_client_functional_crap', 'a').write('CLIENT - done with compile\n')
self.server.compilation_done(buildpath)
self.busy_on = None
@@ -82,7 +87,7 @@
finally:
channel.close()
"""
-def init(gw, sysconfig, path=None, port=12321, testing=False):
+def init(gw, sysconfig, path=None, port=12321, testing_sleeptime=False):
from pypy.tool.build import execnetconference
from pypy.config.config import make_dict
@@ -91,7 +96,8 @@
sysinfo = make_dict(sysconfig)
conference = execnetconference.conference(gw, port, False)
- channel = conference.remote_exec(initcode % (path, sysinfo, testing))
+ channel = conference.remote_exec(initcode % (path, sysinfo,
+ testing_sleeptime))
return channel
class ChannelWrapper(object):
@@ -154,7 +160,7 @@
path=config.path,
port=config.port)
- print channel.receive() # welcome message
+ print 'connected'
try:
try:
while 1:
@@ -204,7 +210,7 @@
try:
print 'starting compilation'
upath, log = compilefunc(svnwc, request.compileinfo)
- except KeyboardInterrupt:
+ except (SystemExit, KeyboardInterrupt):
print 'quitting...'
break
Modified: pypy/dist/pypy/tool/build/conftest.py
==============================================================================
--- pypy/dist/pypy/tool/build/conftest.py (original)
+++ pypy/dist/pypy/tool/build/conftest.py Tue Dec 19 22:52:26 2006
@@ -11,8 +11,6 @@
),
)
-py.test.pypybuilder_option = option
-
class Directory(Dir):
def run(self):
if self.fspath == mypath:
Modified: pypy/dist/pypy/tool/build/server.py
==============================================================================
--- pypy/dist/pypy/tool/build/server.py (original)
+++ pypy/dist/pypy/tool/build/server.py Tue Dec 19 22:52:26 2006
@@ -26,18 +26,22 @@
return True
class PPBServer(object):
+ """ the build server
+
+ this delegates or queues build requests, and stores results and sends
+ out emails when they're done
+ """
retry_interval = 10
def __init__(self, projname, channel, builddir, mailhost=None,
mailport=None, mailfrom=None):
self._projname = projname
self._channel = channel
- self._builddir = builddir
+ self._buildroot = py.path.local(builddir)
self._mailhost = mailhost
self._mailport = mailport
self._mailfrom = mailfrom
- self._buildroot = py.path.local(builddir)
self._clients = []
done = []
@@ -49,6 +53,7 @@
bp.remove()
self._done = done
+
self._queued = [] # no compile client available
self._waiting = [] # compilation already in progress for someone else
@@ -56,10 +61,10 @@
self._namelock = thread.allocate_lock()
def register(self, client):
+ """ register a client (instance) """
self._clients.append(client)
self._channel.send('registered %s with info %r' % (
client, client.sysinfo))
- client.channel.send('welcome')
def compile(self, request):
"""start a compilation
@@ -121,7 +126,10 @@
)
accepted = client.compile(request)
if accepted:
+ self._channel.send('compile job accepted')
return True
+ else:
+ self._channel.send('compile job denied')
self._channel.send(
'no suitable client available for compilation of %s' % (
request,
@@ -167,6 +175,7 @@
clients = self._clients[:]
for client in clients:
if client.channel.isclosed():
+ self._channel.send('client %s disconnected' % (client,))
if client.busy_on:
self._queued.append(client.busy_on)
self._clients.remove(client)
@@ -282,3 +291,4 @@
mailhost, mailport,
mailfrom))
return channel
+
Modified: pypy/dist/pypy/tool/build/test/test_pypybuilder.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_pypybuilder.py (original)
+++ pypy/dist/pypy/tool/build/test/test_pypybuilder.py Tue Dec 19 22:52:26 2006
@@ -1,14 +1,26 @@
+import py
+import time
import path
+
from pypy.tool.build import client, server, execnetconference
from pypy.tool.build import config
from pypy.tool.build import build
+from pypy.tool.build.conftest import option
from pypy.config import config as pypyconfig
-import py
+
from repo import create_temp_repo
+""" some functional tests (although some of the rest aren't strictly
+ unit tests either), to run use --functional as an arg to py.test
+"""
+
+# XXX this one is a bit messy, it's a quick functional test for the whole
+# system, but for instance contains time.sleep()s to make sure all threads
+# get the time to perform tasks and such...
+
# XXX NOTE: if you encounter failing tests on a slow system, you may want to
# increase the sleep interval a bit to see if that helps...
-SLEEP_INTERVAL = 1.0
+SLEEP_INTERVAL = 1.0 # 1 sec default, seems okay even on slow machines
def _get_sysconfig():
return pypyconfig.Config(
@@ -17,47 +29,46 @@
])
)
-# some functional tests (although some of the rest aren't strictly
-# unit tests either), to run use --functional as an arg to py.test
-def test_functional_1():
- if not py.test.pypybuilder_option.functional:
+def setup_module(mod):
+ if not option.functional:
py.test.skip('skipping functional test, use --functional to run it')
+ mod.repo = repo = create_temp_repo('functional')
+ repo.mkdir('foo')
+ mod.foourl = str(repo.join('foo'))
config.checkers = []
- # XXX this one is a bit messy, it's a quick functional test for the whole
- # system, but for instance contains time.sleep()s to make sure all threads
- # get the time to perform tasks and such...
-
- repo = create_temp_repo('functional')
- repo.mkdir('foo')
- foourl = str(repo.join('foo'))
+ mod.sgw = sgw = py.execnet.PopenGateway()
+ mod.temppath = temppath = py.test.ensuretemp('pypybuilder-functional')
- # first initialize a server
- sgw = py.execnet.PopenGateway()
- temppath = py.test.ensuretemp('pypybuilder-functional')
- sc = server.init(sgw, port=config.testport, path=config.testpath,
+ mod.sc = sc = server.init(sgw, port=config.testport, path=config.testpath,
buildpath=str(temppath))
- # give the server some time to wake up
- py.std.time.sleep(SLEEP_INTERVAL)
+ def read():
+ while 1:
+ try:
+ print sc.receive()
+ except EOFError:
+ break
+ py.std.thread.start_new_thread(read, ())
- # then two clients, both with different system info
- sysconfig1 = _get_sysconfig()
- cgw1 = py.execnet.PopenGateway()
- cc1 = client.init(cgw1, sysconfig1, port=config.testport, testing=True)
- cc1.receive() # welcome message
-
- sysconfig2 = _get_sysconfig()
- sysconfig2.foo = 2
- cgw2 = py.execnet.PopenGateway()
- cc2 = client.init(cgw2, sysconfig2, port=config.testport, testing=True)
- cc2.receive() # welcome message
+ # give the server some time to wake up
+ time.sleep(SLEEP_INTERVAL)
- # give the clients some time to register themselves
- py.std.time.sleep(SLEEP_INTERVAL)
+def teardown_module(mod):
+ mod.sc.close()
+ mod.sgw.exit()
+
+def create_client_channel(**conf):
+ cgw = py.execnet.PopenGateway()
+ sysconfig = _get_sysconfig()
+ sysconfig.__dict__.update(conf)
+ channel = client.init(cgw, sysconfig, port=config.testport,
+ testing_sleeptime=SLEEP_INTERVAL * 4)
+ channel.send(True)
+ return cgw, channel
- # now we're going to send some compile jobs
+def compile(**sysconfig):
code = """
import sys
sys.path += %r
@@ -67,101 +78,112 @@
channel.send(ppbserver.compile(%r))
channel.close()
"""
- compgw = py.execnet.PopenGateway()
- compconf = execnetconference.conference(compgw, config.testport)
-
- # we're going to have to closely mimic the bin/client script to avoid
- # freezes (from the app waiting for input)
-
- # this one should fail because there's no client found for foo = 3
- br = build.BuildRequest('foo1 at bar.com', {'foo': 3}, {}, foourl,
- 1, 0)
- compc = compconf.remote_exec(code % (config.testpath, br))
-
- # sorry...
- py.std.time.sleep(SLEEP_INTERVAL)
-
- ret = compc.receive()
- assert not ret[0]
- assert ret[1].find('no suitable client found') > -1
-
- # this one should be handled by client 1
- br = build.BuildRequest('foo2 at bar.com', {'foo': 1}, {}, foourl,
- 1, 0)
- compc = compconf.remote_exec(code % (config.testpath, br))
-
- # client 1 will now send a True to the server to tell it wants to compile
- cc1.send(True)
-
- # and another one
- py.std.time.sleep(SLEEP_INTERVAL)
-
- ret = compc.receive()
- print repr(ret)
- assert not ret[0]
- assert ret[1].find('found a suitable client') > -1
-
- # the messages may take a bit to arrive, too
- py.std.time.sleep(SLEEP_INTERVAL)
-
- # client 1 should by now have received the info to build for
- ret = cc1.receive()
- request = build.BuildRequest.fromstring(ret)
- assert request.sysinfo == {'foo': 1}
-
- # this should have created a package in the temp dir
- assert len(temppath.listdir()) == 1
-
- # now we're going to satisfy the first request by adding a new client
- sysconfig3 = _get_sysconfig()
- sysconfig3.foo = 3
- cgw3 = py.execnet.PopenGateway()
- cc3 = client.init(cgw3, sysconfig3, port=config.testport, testing=True)
+ gw = py.execnet.PopenGateway()
+ conf = execnetconference.conference(gw, config.testport)
- # add True to the buffer just like we did for channels 1 and 2
- cc3.send(True)
+ try:
+ br = build.BuildRequest('foo at bar.com', sysconfig, {}, foourl, 1, 0)
+ channel = conf.remote_exec(code % (config.testpath, br))
+ try:
+ # sorry...
+ time.sleep(SLEEP_INTERVAL)
+ ret = channel.receive()
+ finally:
+ channel.close()
+ finally:
+ gw.exit()
- # again a bit of waiting may be desired
- py.std.time.sleep(SLEEP_INTERVAL)
+ return ret
- # _try_queued() should check whether there are new clients available for
- # queued jobs
- code = """
+def get_info(attr):
+ code = py.code.Source("""
import sys, time
sys.path += %r
from pypy.tool.build import ppbserver
+ ppbserver._cleanup_clients()
+ ppbserver._test_waiting()
ppbserver._try_queued()
- # give the server some time, the clients 'compile' in threads
- time.sleep(%s)
- channel.send(ppbserver._waiting)
- channel.close()
- """
- compgw2 = py.execnet.PopenGateway()
- compconf2 = execnetconference.conference(compgw2, config.testport)
- compc2 = compconf2.remote_exec(code % (config.testpath, SLEEP_INTERVAL))
- cc2.send(True)
+ # take some time to update all the lists
+ time.sleep(%s)
+
+ data = [str(x) for x in ppbserver.%s]
+ channel.send(data)
+ channel.close()
+ """ % (config.testpath, SLEEP_INTERVAL, attr))
+ gw = py.execnet.PopenGateway()
+ try:
+ cf = execnetconference.conference(gw, config.testport)
+ channel = cf.remote_exec(code)
+ try:
+ ret = channel.receive()
+ finally:
+ channel.close()
+ finally:
+ gw.exit()
+ return ret
+
+def test_functional():
+ # first we check if the queues are empty
+ queued = get_info('_queued')
+ assert len(queued) == 0
+ waiting = get_info('_waiting')
+ assert len(waiting) == 0
+ clients = get_info('_clients')
+ assert len(clients) == 0
+
+ # then we request a compilation for sysinfo foo=1, obviously this can not
+ # be fulfilled yet
+ ispath, data = compile(foo=1)
+ assert not ispath
+ assert 'no suitable client' in data
+ queued = get_info('_queued')
+ assert len(queued) == 1
+
+ # now we register a client with the same sysinfo, note that we don't tell
+ # the server yet that the client actually accepts to handle the request
+ gw, cchannel = create_client_channel(foo=1)
+ try:
+ clients = get_info('_clients')
+ assert len(clients) == 1
+
+ # XXX quite a bit scary here, the client will take exactly
+ # 4 * SLEEP_INTERVAL seconds to fake the compilation... here we should
+ # (if all is well) still be compiling
+
+ ispath, data = compile(foo=1)
+ assert not ispath
+ assert 'in progress' in data
+
+ waiting = get_info('_waiting')
+ assert len(waiting) == 1
+
+ # this sleep, along with that in the previous compile call, should be
+ # enough to reach the end of fake compilation
+ time.sleep(SLEEP_INTERVAL * 3)
+
+ # both the jobs should have been done now...
+ queued = get_info('_queued')
+ assert len(queued) == 0
+
+ waiting = get_info('_waiting')
+ assert len(waiting) == 0
+
+ # now a new request for the same build should return in a path being
+ # returned
+ ispath, data = compile(foo=1)
+ assert ispath
+
+ queued = get_info('_queued')
+ assert len(queued) == 0
+ waiting = get_info('_waiting')
+ assert len(waiting) == 0
+
+ finally:
+ cchannel.close()
+ gw.exit()
- # we check whether all emails are now sent, since after adding the third
- # client, and calling _try_queued(), both jobs should have been processed
- ret = compc2.receive()
- assert ret == []
-
- # this should also have created another package in the temp dir
- assert len(temppath.listdir()) == 2
-
- # some cleanup (this should all be in nested try/finallys, blegh)
- cc1.close()
- cc2.close()
- cc3.close()
- compc.close()
- compc2.close()
- sc.close()
-
- cgw1.exit()
- cgw2.exit()
- compgw.exit()
- compgw2.exit()
- sgw.exit()
+ clients = get_info('_clients')
+ assert len(clients) == 0
Modified: pypy/dist/pypy/tool/build/test/test_server.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_server.py (original)
+++ pypy/dist/pypy/tool/build/test/test_server.py Tue Dec 19 22:52:26 2006
@@ -36,8 +36,6 @@
assert svr._clients[0] == c1
assert svr._clients[1] == c2
- assert c1.channel.receive() == 'welcome'
- assert c2.channel.receive() == 'welcome'
py.test.raises(IndexError, "c1.channel.receive()")
assert svr._channel.receive().find('registered') > -1
More information about the Pypy-commit
mailing list