[pypy-svn] r30675 - in pypy/dist/pypy/tool/build: . bin test
guido at codespeak.net
guido at codespeak.net
Fri Jul 28 12:20:07 CEST 2006
Author: guido
Date: Fri Jul 28 12:19:53 2006
New Revision: 30675
Modified:
pypy/dist/pypy/tool/build/bin/client
pypy/dist/pypy/tool/build/bin/startcompile
pypy/dist/pypy/tool/build/client.py
pypy/dist/pypy/tool/build/config.py
pypy/dist/pypy/tool/build/server.py
pypy/dist/pypy/tool/build/test/fake.py
pypy/dist/pypy/tool/build/test/test_client.py
pypy/dist/pypy/tool/build/test/test_pypybuilder.py
pypy/dist/pypy/tool/build/test/test_request_storage.py
pypy/dist/pypy/tool/build/test/test_server.py
Log:
Unsing PyPy's config mechanism now for system info and cmdline args.
Modified: pypy/dist/pypy/tool/build/bin/client
==============================================================================
--- pypy/dist/pypy/tool/build/bin/client (original)
+++ pypy/dist/pypy/tool/build/bin/client Fri Jul 28 12:19:53 2006
@@ -7,34 +7,18 @@
import random
from pypy.tool.build import config
-# XXX using random values for testing
-modules = ['_stackless', '_socket']
-
-"""
-random.shuffle(modules)
-sysinfo = {
- 'maxint': random.choice((sys.maxint, (2 ** 63 - 1))),
- 'use_modules': modules[:random.randrange(len(modules) + 1)],
- 'byteorder': random.choice(('little', 'big')),
-}
-"""
-
-sysinfo = {
- 'maxint': sys.maxint,
- 'use_modules': ['_stackless', '_socket'],
- 'byteorder': sys.byteorder,
-}
-
if __name__ == '__main__':
from py.execnet import SshGateway
from pypy.tool.build.client import init
gw = SshGateway(config.server)
- channel = init(gw, sysinfo, path=config.path, port=config.port)
+ channel = init(gw, config.system_config, path=config.path,
+ port=config.port)
+
print channel.receive() # welcome message
try:
while 1:
data = channel.receive()
- if not isinstance(data, dict): # needs more checks here
+ if not isinstance(data, tuple): # needs more checks here
raise ValueError(
'received wrong unexpected data of type %s' % (type(data),)
)
Modified: pypy/dist/pypy/tool/build/bin/startcompile
==============================================================================
--- pypy/dist/pypy/tool/build/bin/startcompile (original)
+++ pypy/dist/pypy/tool/build/bin/startcompile Fri Jul 28 12:19:53 2006
@@ -5,47 +5,64 @@
import random
from pypy.tool.build import config
+def parse_options(config):
+ # merge system + compile options into one optionparser
+ from optparse import OptionParser, OptionGroup
+ from pypy.config.config import to_optparse
+
+ optparser = OptionParser()
+ sog = OptionGroup(optparser, 'system', 'System options')
+ system_options = to_optparse(config.system_config,
+ config.system_config.getpaths())
+ [sog.add_option(o) for o in system_options.option_list[1:]]
+ optparser.add_option_group(sog)
+
+ cog = OptionGroup(optparser, 'compile', 'Compile options')
+ compile_options = to_optparse(config.compile_config,
+ config.compile_config.getpaths())
+ [cog.add_option(o) for o in compile_options.option_list[1:]]
+ optparser.add_option_group(cog)
+
+ (options, args) = optparser.parse_args()
+
+ if not args or len(args) != 1:
+ optparser.error('please provide an email address')
+
+ return optparser, options, args
+
initcode = """
import sys
sys.path += %r
from pypy.tool.build import ppbserver
- channel.send(ppbserver.compile(%r, %r))
+ channel.send(ppbserver.compile(%r, (%r, %r)))
channel.close()
"""
-def init(gw, sysinfo, email, port=12321):
+def init(gw, sysinfo, compileinfo, path, email, port=12321):
from pypy.tool.build import execnetconference
conference = execnetconference.conference(gw, port, False)
- channel = conference.remote_exec(initcode % (config.path, email, sysinfo))
+ channel = conference.remote_exec(initcode % (path, email,
+ sysinfo, compileinfo))
return channel
if __name__ == '__main__':
from py.execnet import SshGateway
+ from pypy.tool.build.server import config_to_dict
- from optparse import OptionParser
- optparser = OptionParser('%prog [options] email')
- for args, kwargs in config.options:
- optparser.add_option(*args, **kwargs)
- optparser.add_option('-r', '--revision', dest='revision', default='trunk',
- help='SVN revision (defaults to "trunk")')
-
- (options, args) = optparser.parse_args()
+ optparser, options, args = parse_options(config)
- if not args or len(args) != 1:
- optparser.error('please provide an email address')
+ sysinfo = config_to_dict(config.system_config)
+ compileinfo = config_to_dict(config.compile_config)
- sysinfo = dict([(attr, getattr(options, attr)) for attr in dir(options) if
- not attr.startswith('_') and
- not callable(getattr(options, attr))])
-
print 'going to start compile job with info:'
for k, v in sysinfo.items():
print '%s: %r' % (k, v)
print
gw = SshGateway(config.server)
- channel = init(gw, sysinfo, args[0], port=config.port)
+ channel = init(gw, sysinfo, compileinfo, config.path, args[0],
+ port=config.port)
ispath, data = channel.receive()
if ispath:
print ('a suitable result is already available, you can find it '
Modified: pypy/dist/pypy/tool/build/client.py
==============================================================================
--- pypy/dist/pypy/tool/build/client.py (original)
+++ pypy/dist/pypy/tool/build/client.py Fri Jul 28 12:19:53 2006
@@ -18,20 +18,21 @@
self.channel.close()
def compile(self, info):
- """send a compile job to the client side
-
- this waits until the client is done, and assumes the client sends
- back the whole binary as a single string (XXX this should change ;)
- """
+ """send a compile job to the client side"""
self.busy_on = info
self.channel.send(info)
thread.start_new_thread(self.wait_until_done, (info,))
def wait_until_done(self, info):
+ efp = open('/tmp/foo', 'w')
+ efp.write(repr(info) + '\n')
buildpath = self.server.get_new_buildpath(info)
+ efp.flush()
- fp = buildpath.zipfile.open('w')
if not self.testing:
+ efp.write('2\n')
+ efp.flush()
+ fp = buildpath.zipfile.open('w')
try:
while True:
try:
@@ -44,9 +45,14 @@
fp.write(chunk)
finally:
fp.close()
-
+
+ efp.write('3\n')
+ efp.flush()
self.server.compilation_done(info, buildpath)
self.busy_on = None
+ efp.write(repr(info))
+ efp.flush()
+ efp.close()
initcode = """
import sys
@@ -60,12 +66,14 @@
finally:
channel.close()
"""
-def init(gw, sysinfo, path=None, port=12321, testing=False):
+def init(gw, sysconfig, path=None, port=12321, testing=False):
from pypy.tool.build import execnetconference
+ from pypy.tool.build import server
if path is None:
path = []
+ sysinfo = server.config_to_dict(sysconfig)
conference = execnetconference.conference(gw, port, False)
channel = conference.remote_exec(initcode % (path, sysinfo, testing))
return channel
Modified: pypy/dist/pypy/tool/build/config.py
==============================================================================
--- pypy/dist/pypy/tool/build/config.py (original)
+++ pypy/dist/pypy/tool/build/config.py Fri Jul 28 12:19:53 2006
@@ -5,33 +5,19 @@
port = 12321
path = ['/home/johnny/temp/pypy-dist']
-# option definitions for the startcompile script
-# for now we have them here, we should probably use pypy's config instead
-# though...
-import sys
-def _use_modules_callback(option, opt_str, value, parser):
- parser.values.use_modules = [m.strip() for m in value.split(',')
- if m.strip()]
-
-def _maxint_callback(option, opt_str, value, parser):
- parser.values.maxint = 2 ** (int(value) - 1) - 1
-
-options = [
- (('-m', '--use-modules'), {'action': 'callback', 'type': 'string',
- 'callback': _use_modules_callback,
- 'dest': 'use_modules', 'default': [],
- 'help': 'select the modules you want to use'}),
- (('-i', '--maxint'), {'action': 'callback', 'callback': _maxint_callback,
- 'default': sys.maxint, 'dest': 'maxint',
- 'type': 'string',
- 'help': ('size of an int in bits (32/64, '
- 'defaults to sys.maxint)')}),
- (('-b', '--byteorder'), {'action': 'store',
- 'dest': 'byteorder', 'default': sys.byteorder,
- 'nargs': 1,
- 'help': ('byte order (little/big, defaults '
- 'to sys.byteorder)')}),
-]
+# configuration of options for client and startcompile
+from pypy.config.config import Config, to_optparse
+
+# system config, on the client everything is set by scanning the system, when
+# calling startcompile defaults are taken from the system, overrides are
+# possible using cmdline args
+from systemoption import system_optiondescription
+system_config = Config(system_optiondescription)
+
+# compile option config, used by client to parse info, by startcompile for
+# cmdline args, defaults are taken from the optiondescription
+from pypy.config.pypyoption import pypy_optiondescription
+compile_config = Config(pypy_optiondescription)
# settings for the server
projectname = 'pypy'
Modified: pypy/dist/pypy/tool/build/server.py
==============================================================================
--- pypy/dist/pypy/tool/build/server.py (original)
+++ pypy/dist/pypy/tool/build/server.py Fri Jul 28 12:19:53 2006
@@ -24,8 +24,17 @@
return False
return True
-# XXX note that all this should be made thread-safe at some point (meaning it
-# currently isn't)!!
+def config_to_dict(config, is_optiondescription=False):
+ from pypy.config.config import OptionDescription
+ ret = {}
+ children = config._descr._children
+ for child in children:
+ value = getattr(config, child._name)
+ if isinstance(child, OptionDescription):
+ ret[child._name] = config_to_dict(value, True)
+ else:
+ ret[child._name] = value
+ return ret
class RequestStorage(object):
"""simple registry that manages information"""
@@ -45,7 +54,6 @@
this either returns a path to the binary (if it's available
already) or an id for the info
"""
- self._normalize(info)
infoid = self.get_info_id(info)
path = self._id_to_path.get(infoid)
if path is not None:
@@ -56,7 +64,6 @@
"""retrieve or create an id for an info dict"""
self._id_lock.acquire()
try:
- self._normalize(info)
for k, v in self._id_to_info.iteritems():
if v == info:
return k
@@ -73,7 +80,6 @@
returns a list of email addresses for the people that should be
warned
"""
- self._normalize(info)
infoid = self.get_info_id(info)
emails = self._id_to_emails.pop(infoid)
self._id_to_path[infoid] = path
@@ -85,41 +91,42 @@
id = self.get_info_id(info)
self._id_to_path[id] = path
- def _normalize(self, info):
- for k, v in info.iteritems():
- if isinstance(v, list):
- v.sort()
-
from py.__.path.local.local import LocalPath
class BuildPath(LocalPath):
def _info(self):
- info = getattr(self, '_info_value', {})
+ info = getattr(self, '_info_value', [])
if info:
return info
- infopath = self / 'info.txt'
- if not infopath.check():
- return {}
- for line in infopath.readlines():
- line = line.strip()
- if not line:
- continue
- chunks = line.split(':')
- key = chunks.pop(0)
- value = ':'.join(chunks)
- info[key] = eval(value)
+ for name in ['system', 'compile']:
+ currinfo = {}
+ infopath = self.join('%s_info.txt' % (name,))
+ if not infopath.check():
+ return ({}, {})
+ for line in infopath.readlines():
+ line = line.strip()
+ if not line:
+ continue
+ chunks = line.split(':')
+ key = chunks.pop(0)
+ value = ':'.join(chunks)
+ currinfo[key] = eval(value)
+ info.append(currinfo)
+ info = tuple(info)
self._info_value = info
return info
def _set_info(self, info):
self._info_value = info
- infopath = self / 'info.txt'
- infopath.ensure()
- fp = infopath.open('w')
- try:
- for key, value in info.iteritems():
- fp.write('%s: %r\n' % (key, value))
- finally:
- fp.close()
+ assert len(info) == 2, 'not a proper info tuple'
+ for i, name in enumerate(['system', 'compile']):
+ infopath = self.join('%s_info.txt' % (name,))
+ infopath.ensure()
+ fp = infopath.open('w')
+ try:
+ for key, value in info[i].iteritems():
+ fp.write('%s: %r\n' % (key, value))
+ finally:
+ fp.close()
info = property(_info, _set_info)
@@ -168,6 +175,10 @@
def compile(self, requester_email, info):
"""start a compilation
+ requester_email is an email address of the person requesting the
+ build, info is a tuple (sysinfo, compileinfo) where both infos
+ are configs converted (or serialized, basically) to dict
+
returns a tuple (ispath, data)
if there's already a build available for info, this will return
@@ -203,15 +214,10 @@
# XXX shuffle should be replaced by something smarter obviously ;)
clients = self._clients[:]
random.shuffle(clients)
- rev = info.pop('revision', 'trunk')
for client in clients:
- # popping out revision here, going to add later... the client
- # should be able to retrieve source code for any revision (so
- # it doesn't need to match a revision field in client.sysinfo)
- if client.busy_on or not issubdict(info, client.sysinfo):
+ if client.busy_on or not issubdict(info[0], client.sysinfo):
continue
else:
- info['revision'] = rev
self._channel.send(
'going to send compile job with info %r to %s' % (
info, client
@@ -219,7 +225,6 @@
)
client.compile(info)
return True
- info['revision'] = rev
self._channel.send(
'no suitable client available for compilation with info %r' % (
info,
Modified: pypy/dist/pypy/tool/build/test/fake.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/fake.py (original)
+++ pypy/dist/pypy/tool/build/test/fake.py Fri Jul 28 12:19:53 2006
@@ -23,8 +23,7 @@
self.busy_on = None
def compile(self, info):
- info.pop('revision')
- for k, v in info.items():
+ for k, v in info[0].items():
self.channel.send('%s: %r' % (k, v))
self.channel.send(None)
self.busy_on = info
Modified: pypy/dist/pypy/tool/build/test/test_client.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_client.py (original)
+++ pypy/dist/pypy/tool/build/test/test_client.py Fri Jul 28 12:19:53 2006
@@ -25,7 +25,7 @@
svr.register(c2)
def test_compile():
- info = {'foo': 1}
+ info = ({'foo': 1}, {'bar': 2})
c1.compile(info)
c1.channel.receive()
c1.channel.send('foo bar')
Modified: pypy/dist/pypy/tool/build/test/test_pypybuilder.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_pypybuilder.py (original)
+++ pypy/dist/pypy/tool/build/test/test_pypybuilder.py Fri Jul 28 12:19:53 2006
@@ -1,8 +1,16 @@
import path
from pypy.tool.build import client, server, execnetconference
from pypy.tool.build import config
+from pypy.config import config as pypyconfig
import py
+def _get_sysconfig():
+ return pypyconfig.Config(
+ pypyconfig.OptionDescription('foo', [
+ pypyconfig.ChoiceOption('foo', 'foo', [1,2,3], 1),
+ ])
+ )
+
# some functional tests (although some of the rest aren't strictly
# unit tests either), to run use --functional as an arg to py.test
def test_functional_1():
@@ -25,19 +33,14 @@
py.std.time.sleep(sleep_interval)
# then two clients, both with different system info
- sysinfo1 = {
- 'foo': 1,
- 'bar': [1,2],
- }
+ sysconfig1 = _get_sysconfig()
cgw1 = py.execnet.PopenGateway()
- cc1 = client.init(cgw1, sysinfo1, port=config.port, testing=True)
+ cc1 = client.init(cgw1, sysconfig1, port=config.port, testing=True)
- sysinfo2 = {
- 'foo': 2,
- 'bar': [1],
- }
+ sysconfig2 = _get_sysconfig()
+ sysconfig2.foo = 2
cgw2 = py.execnet.PopenGateway()
- cc2 = client.init(cgw2, sysinfo2, port=config.port, testing=True)
+ cc2 = client.init(cgw2, sysconfig2, port=config.port, testing=True)
# give the clients some time to register themselves
py.std.time.sleep(sleep_interval)
@@ -48,7 +51,7 @@
sys.path += %r
from pypy.tool.build import ppbserver
- channel.send(ppbserver.compile(%r, %r))
+ channel.send(ppbserver.compile(%r, (%r, {})))
channel.close()
"""
compgw = py.execnet.PopenGateway()
@@ -67,7 +70,7 @@
# this one should be handled by client 1
compc = compconf.remote_exec(code % (config.testpath, 'foo2 at bar.com',
- {'foo': 1, 'bar': [1]}))
+ {'foo': 1}))
# and another one
py.std.time.sleep(sleep_interval)
@@ -82,15 +85,16 @@
# client 1 should by now have received the info to build for
cc1.receive() # 'welcome'
ret = cc1.receive()
- assert ret == {'foo': 1, 'bar': [1], 'revision': 'trunk'}
+ assert ret == ({'foo': 1}, {})
# this should have created a package in the temp dir
assert len(temppath.listdir()) == 1
# now we're going to satisfy the first request by adding a new client
- sysinfo3 = {'foo': 3}
+ sysconfig3 = _get_sysconfig()
+ sysconfig3.foo = 3
cgw3 = py.execnet.PopenGateway()
- cc3 = client.init(cgw3, sysinfo3, port=config.port, testing=True)
+ cc3 = client.init(cgw3, sysconfig3, port=config.port, testing=True)
# again a bit of waiting may be desired
py.std.time.sleep(sleep_interval)
Modified: pypy/dist/pypy/tool/build/test/test_request_storage.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_request_storage.py (original)
+++ pypy/dist/pypy/tool/build/test/test_request_storage.py Fri Jul 28 12:19:53 2006
@@ -58,7 +58,3 @@
assert s._id_to_emails == {}
assert s._id_to_path == {id1: 'foo', id2: 'bar'}
-def test__normalize():
- s = RequestStorage()
- assert (s._normalize({'foo': ['bar', 'baz']}) ==
- s._normalize({'foo': ['baz', 'bar']}))
Modified: pypy/dist/pypy/tool/build/test/test_server.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_server.py (original)
+++ pypy/dist/pypy/tool/build/test/test_server.py Fri Jul 28 12:19:53 2006
@@ -47,7 +47,7 @@
def test_compile():
# XXX this relies on the output not changing... quite scary
info = {'foo': 1}
- ret = svr.compile('test at domain.com', info)
+ ret = svr.compile('test at domain.com', (info, None))
assert not ret[0]
assert ret[1].find('found a suitable client') > -1
assert svr._channel.receive().find('going to send compile job') > -1
@@ -55,26 +55,26 @@
assert c1.channel.receive() is None
py.test.raises(IndexError, "c2.channel.receive()")
- svr.compile('test at domain.com', {'foo': 3})
+ svr.compile('test at domain.com', ({'foo': 3}, None))
assert svr._channel.receive().find('no suitable client available') > -1
info = {'bar': [3]}
- ret = svr.compile('test at domain.com', info)
+ ret = svr.compile('test at domain.com', (info, None))
assert svr._channel.receive().find('going to send') > -1
assert c2.channel.receive() == 'bar: [3]'
assert c2.channel.receive() is None
py.test.raises(IndexError, "c1.channel.receive()")
info = {'foo': 1}
- ret = svr.compile('test at domain.com', info)
+ ret = svr.compile('test at domain.com', (info, None))
assert not ret[0]
assert ret[1].find('this build is already') > -1
assert svr._channel.receive().find('currently in progress') > -1
c1.busy_on = None
bp = BuildPath(str(temppath / 'foo'))
- svr.compilation_done(info, bp)
- ret = svr.compile('test at domain.com', info)
+ svr.compilation_done((info, None), bp)
+ ret = svr.compile('test at domain.com', (info, None))
assert ret[0]
assert isinstance(ret[1], BuildPath)
assert ret[1] == bp
@@ -88,11 +88,11 @@
# grmbl... local.__new__ checks for class equality :(
bp = BuildPath(str(tempdir / 'test1'))
assert not bp.check()
- assert bp.info == {}
+ assert bp.info == ({}, {})
- bp.info = {'foo': 1, 'bar': [1,2]}
- assert bp.info == {'foo': 1, 'bar': [1,2]}
- assert (sorted((bp / 'info.txt').readlines()) ==
+ bp.info = ({'foo': 1, 'bar': [1,2]}, {'baz': 1})
+ assert bp.info == ({'foo': 1, 'bar': [1,2]}, {'baz': 1})
+ assert (sorted((bp / 'system_info.txt').readlines()) ==
['bar: [1, 2]\n', 'foo: 1\n'])
assert isinstance(bp.zipfile, py.path.local)
@@ -116,15 +116,15 @@
svr._i = 0
today = time.strftime('%Y%m%d')
- path1 = svr.get_new_buildpath({'foo': 'bar'})
+ path1 = svr.get_new_buildpath(({'foo': 'bar'}, {'baz': 'qux'}))
try:
assert isinstance(path1, BuildPath)
- assert path1.info == {'foo': 'bar'}
+ assert path1.info == ({'foo': 'bar'}, {'baz': 'qux'})
assert path1.basename == 'pypytest-%s-0' % (today,)
try:
- path2 = svr.get_new_buildpath({'foo': 'baz'})
- assert path2.info == {'foo': 'baz'}
+ path2 = svr.get_new_buildpath(({'foo': 'baz'}, {'bar': 'qux'}))
+ assert path2.info == ({'foo': 'baz'}, {'bar': 'qux'})
assert path2.basename == 'pypytest-%s-1' % (today,)
finally:
path2.remove()
More information about the Pypy-commit
mailing list