[pypy-commit] buildbot default: merge issue-1759 which moves testing runs to <tmp>/pytest and transfers numpy-compatability test to triggered builds
mattip
noreply at buildbot.pypy.org
Tue Mar 3 17:47:39 CET 2015
Author: mattip <matti.picus at gmail.com>
Branch:
Changeset: r936:9368e32b1675
Date: 2015-03-03 18:49 +0200
http://bitbucket.org/pypy/buildbot/changeset/9368e32b1675/
Log: merge issue-1759 which moves testing runs to <tmp>/pytest and
transfers numpy-compatability test to triggered builds
diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py
--- a/bot2/pypybuildbot/arm_master.py
+++ b/bot2/pypybuildbot/arm_master.py
@@ -1,5 +1,5 @@
from buildbot.scheduler import Nightly, Triggerable
-from pypybuildbot.util import we_are_debugging, load
+from pypybuildbot.util import load
pypybuilds = load('pypybuildbot.builds')
ARMCrossLock = pypybuilds.ARMCrossLock
diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -4,7 +4,7 @@
from buildbot.process import factory
from buildbot.steps import shell, transfer
from buildbot.steps.trigger import Trigger
-from buildbot.process.properties import WithProperties
+from buildbot.process.properties import WithProperties, Interpolate
from buildbot import locks
from pypybuildbot.util import symlink_force
from buildbot.status.results import SKIPPED, SUCCESS
@@ -326,14 +326,14 @@
workdir=workdir,
logEnviron=False))
-def update_git(platform, factory, repourl, workdir, use_branch,
- force_branch=None):
+def update_git(platform, factory, repourl, workdir, branch='master'):
factory.addStep(
Git(
repourl=repourl,
mode='full',
method='fresh',
workdir=workdir,
+ branch=branch,
logEnviron=False))
def setup_steps(platform, factory, workdir=None,
@@ -378,6 +378,35 @@
return ".tar.bz2"
def add_translated_tests(factory, prefix, platform, app_tests, lib_python, pypyjit):
+ factory.addStep(shell.SetPropertyFromCommand(
+ command=['python', '-c', "import tempfile, os ;print"
+ " tempfile.gettempdir() + os.path.sep"],
+ property="target_tmpdir"))
+ # If target_tmpdir is empty, crash.
+ tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s'
+ pytest = "pytest"
+ factory.addStep(PytestCmd(
+ description="mkdir for tests",
+ command=['python', '-c', Interpolate("import os; os.mkdir(r'" + \
+ tmp_or_crazy + pytest + "') if not os.path.exists(r'" + \
+ tmp_or_crazy + pytest + "') else True")],
+ haltOnFailure=True,
+ ))
+
+ nDays = '3' #str, not int
+ if platform == 'win32':
+ command = ['FORFILES', '/P', Interpolate(tmp_or_crazy + pytest),
+ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"]
+ else:
+ command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime',
+ '+' + nDays, '-exec', 'rm', '{}', ';']
+ factory.addStep(PytestCmd(
+ description="cleanout old test files",
+ command = command,
+ flunkOnFailure=False,
+ haltOnFailure=False,
+ ))
+
if app_tests:
if app_tests == True:
app_tests = []
@@ -392,7 +421,9 @@
] + ["--config=%s" % cfg for cfg in app_tests],
logfiles={'pytestLog': 'pytest-A.log'},
timeout=4000,
- env={"PYTHONPATH": ['.']}))
+ env={"PYTHONPATH": ['.'],
+ "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
+ }))
if lib_python:
factory.addStep(PytestCmd(
@@ -402,7 +433,9 @@
"--timeout=3600",
"--resultlog=cpython.log", "lib-python"],
timeout=4000,
- logfiles={'pytestLog': 'cpython.log'}))
+ logfiles={'pytestLog': 'cpython.log'},
+ env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
+ }))
if pypyjit:
# kill this step when the transition to test_pypy_c_new has been
@@ -414,7 +447,9 @@
"--pypy=pypy/goal/pypy-c",
"--resultlog=pypyjit.log",
"pypy/module/pypyjit/test"],
- logfiles={'pytestLog': 'pypyjit.log'}))
+ logfiles={'pytestLog': 'pypyjit.log'},
+ env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
+ }))
#
# "new" test_pypy_c
if platform == 'win32':
@@ -426,7 +461,10 @@
command=prefix + [cmd, "pypy/test_all.py",
"--resultlog=pypyjit_new.log",
"pypy/module/pypyjit/test_pypy_c"],
- logfiles={'pytestLog': 'pypyjit_new.log'}))
+ logfiles={'pytestLog': 'pypyjit_new.log'},
+ env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
+ }))
+
# ----
@@ -438,6 +476,36 @@
setup_steps(platform, self)
timeout=kwargs.get('timeout', 4000)
+
+ self.addStep(shell.SetPropertyFromCommand(
+ command=['python', '-c', "import tempfile, os ;print"
+ " tempfile.gettempdir() + os.path.sep"],
+ property="target_tmpdir"))
+ # If target_tmpdir is empty, crash.
+ tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s'
+ pytest = "pytest"
+ self.addStep(PytestCmd(
+ description="mkdir for tests",
+ command=['python', '-c', Interpolate("import os; os.mkdir(r'" + \
+ tmp_or_crazy + pytest + "') if not os.path.exists(r'" + \
+ tmp_or_crazy + pytest + "') else True")],
+ haltOnFailure=True,
+ ))
+
+ nDays = '3' #str, not int
+ if platform == 'win32':
+ command = ['FORFILES', '/P', Interpolate(tmp_or_crazy + pytest),
+ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"]
+ else:
+ command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime',
+ '+' + nDays, '-exec', 'rm', '{}', ';']
+ self.addStep(PytestCmd(
+ description="cleanout old test files",
+ command = command,
+ flunkOnFailure=False,
+ haltOnFailure=False,
+ ))
+
self.addStep(PytestCmd(
description="pytest pypy",
command=["python", "testrunner/runner.py",
@@ -449,7 +517,9 @@
logfiles={'pytestLog': 'testrun.log'},
timeout=timeout,
env={"PYTHONPATH": ['.'],
- "PYPYCHERRYPICK": cherrypick}))
+ "PYPYCHERRYPICK": cherrypick,
+ "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
+ }))
self.addStep(PytestCmd(
description="pytest rpython",
@@ -462,7 +532,9 @@
logfiles={'pytestLog': 'testrun.log'},
timeout=timeout,
env={"PYTHONPATH": ['.'],
- "PYPYCHERRYPICK": cherrypick}))
+ "PYPYCHERRYPICK": cherrypick,
+ "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
+ }))
class Translated(factory.BuildFactory):
@@ -473,7 +545,8 @@
interpreter='pypy',
lib_python=False,
pypyjit=False,
- prefix=None
+ prefix=None,
+ trigger=None,
):
factory.BuildFactory.__init__(self)
if prefix is not None:
@@ -502,6 +575,9 @@
workdir='.',
blocksize=100 * 1024))
+ if trigger: # if provided trigger schedulers that depend on this one
+ self.addStep(Trigger(schedulerNames=[trigger]))
+
add_translated_tests(self, prefix, platform, app_tests, lib_python, pypyjit)
@@ -839,7 +915,6 @@
'''
def __init__(self, platform='linux',
app_tests=False,
- host = 'tannit',
lib_python=False,
pypyjit=True,
prefix=None,
@@ -850,9 +925,18 @@
self.addStep(ParseRevision(hideStepIf=ParseRevision.hideStepIf,
doStepIf=ParseRevision.doStepIf))
# download corresponding nightly build
+ if platform == 'win32':
+ target = r'pypy-c\pypy.exe'
+ untar = ['unzip']
+ sep = '\\'
+ else:
+ target = r'pypy-c/bin/pypy'
+ untar = ['tar', '--strip-components=1', '--directory=.', '-xf']
+ sep = '/'
self.addStep(ShellCmd(
- description="Clear pypy-c",
- command=['rm', '-rf', 'pypy-c'],
+ description="Clear",
+ # assume, as part of git, that windows has rm
+ command=['rm', '-rf', 'pypy-c', 'install'],
workdir='.'))
extension = get_extension(platform)
name = build_name(platform, pypyjit, translationArgs, placeholder='%(final_file_name)s') + extension
@@ -863,12 +947,17 @@
workdir='pypy-c'))
# extract downloaded file
- if platform.startswith('win'):
- raise NotImplementedError
- else:
+ self.addStep(ShellCmd(
+ description="decompress pypy-c",
+ command=untar + ['pypy_build'+ extension],
+ workdir='pypy-c',
+ haltOnFailure=True,
+ ))
+
+ if platform == 'win32':
self.addStep(ShellCmd(
- description="decompress pypy-c",
- command=['tar', '--extract', '--file=pypy_build'+ extension, '--strip-components=1', '--directory=.'],
+ description='move decompressed dir',
+ command = ['mv', '*/*', '.'],
workdir='pypy-c',
haltOnFailure=True,
))
@@ -876,39 +965,45 @@
# virtualenv the download
self.addStep(ShellCmd(
description="create virtualenv",
- command=['virtualenv','-p', 'pypy-c/bin/pypy', 'install'],
+ command=['virtualenv','-p', target, 'install'],
+ workdir='./',
+ haltOnFailure=True,
+ ))
+
+ self.addStep(ShellCmd(
+ description="report version",
+ command=[sep.join(['install','bin','pypy'])] + ['--version'],
workdir='./',
haltOnFailure=True,
))
self.addStep(ShellCmd(
description="install nose",
- command=['install/bin/pip', 'install','nose'],
+ command=[sep.join(['install','bin','pip'])] + ['install','nose'],
workdir='./',
haltOnFailure=True,
))
# obtain a pypy-compatible branch of numpy
numpy_url = 'https://www.bitbucket.org/pypy/numpy'
- numpy_pypy_branch = 'pypy-compat'
- update_git(platform, self, numpy_url, 'numpy_src', use_branch=True,
- force_branch=numpy_pypy_branch)
+ update_git(platform, self, numpy_url, 'numpy_src', branch='master')
self.addStep(ShellCmd(
description="install numpy",
- command=['../install/bin/python', 'setup.py','install'],
+ command=[sep.join(['..', 'install', 'bin', 'pypy'])] + ['setup.py','install'],
workdir='numpy_src'))
self.addStep(ShellCmd(
description="test numpy",
- command=['bin/nosetests', 'site-packages/numpy',
+ command=[sep.join(['bin', 'nosetests'])] + ['site-packages/numpy',
+ # XXX enable '-with-doctest',
],
#logfiles={'pytestLog': 'pytest-numpy.log'},
timeout=4000,
workdir='install',
#env={"PYTHONPATH": ['download']}, # shouldn't be needed, but what if it is set externally?
))
- if host == 'tannit':
+ if platform != 'win32':
self.addStep(ShellCmd(
description="install jinja2",
command=['install/bin/pip', 'install', 'jinja2'],
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -1,6 +1,6 @@
import os
-from buildbot.scheduler import Nightly
+from buildbot.scheduler import Nightly, Triggerable
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.forcesched import ValidationError
from buildbot.buildslave import BuildSlave
@@ -96,6 +96,7 @@
pypyjit=True,
app_tests=True,
platform='linux64',
+ trigger='NUMPY64_scheduler',
)
pypyJITTranslatedTestFactoryIndiana = pypybuilds.Translated(
@@ -133,6 +134,7 @@
lib_python=True,
pypyjit=True,
app_tests=True,
+ trigger='NUMPYWIN_scheduler',
)
pypyJITTranslatedTestFactoryFreeBSD = pypybuilds.Translated(
@@ -144,14 +146,17 @@
app_tests=True,
)
-pypyJITBenchmarkFactory_tannit = pypybuilds.JITBenchmark()
+pypyJITBenchmarkFactory_tannit = pypybuilds.JITBenchmark(host='tannit')
pypyJITBenchmarkFactory64_tannit = pypybuilds.JITBenchmark(platform='linux64',
+ host='tannit',
postfix='-64')
pypyJITBenchmarkFactory64_speed = pypybuilds.JITBenchmarkSingleRun(
platform='linux64',
+ host='speed_python',
postfix='-64')
pypyNumpyCompatability = pypybuilds.NativeNumpyTests(platform='linux64')
+pypyNumpyCompatabilityWin = pypybuilds.NativeNumpyTests(platform='win32')
#
@@ -186,6 +191,7 @@
JITBENCH64_NEW = 'jit-benchmark-linux-x86-64-single-run'
CPYTHON_64 = "cpython-2-benchmark-x86-64"
NUMPY_64 = "numpy-compatability-linux-x86-64"
+NUMPY_WIN = "numpy-compatability-win-x86-32"
# buildbot builder
PYPYBUILDBOT = 'pypy-buildbot'
@@ -216,8 +222,8 @@
APPLVLLINUX64, # on allegro64, uses 1 core
# other platforms
#MACOSX32, # on minime
- JITWIN32, # on aurora, SalsaSalsa
- WIN32, # on aurora, SalsaSalsa
+ JITWIN32, # on allegro_win32, SalsaSalsa
+ WIN32, # on allegro_win32, SalsaSalsa
#JITFREEBSD764, # on headless
#JITFREEBSD864, # on ananke
JITFREEBSD964, # on tavendo
@@ -227,14 +233,20 @@
], branch='default', hour=0, minute=0),
Nightly("nightly-1-00", [
- NUMPY_64, # on tannit64, uses 1 core, takes about 15min.
- # XXX maybe use a trigger instead?
JITBENCH, # on tannit32, uses 1 core (in part exclusively)
JITBENCH64, # on tannit64, uses 1 core (in part exclusively)
JITBENCH64_NEW, # on speed64, uses 1 core (in part exclusively)
], branch=None, hour=1, minute=0),
+ Triggerable("NUMPY64_scheduler", [
+ NUMPY_64, # on tannit64, uses 1 core, takes about 15min.
+ ]),
+
+ Triggerable("NUMPYWIN_scheduler", [
+ NUMPY_WIN, # on allegro_win32, SalsaSalsa
+ ]),
+
Nightly("nightly-2-00-py3k", [
LINUX64, # on allegro64, uses all cores
APPLVLLINUX64, # on allegro64, uses 1 core
@@ -279,6 +291,7 @@
JITBENCH64,
JITBENCH64_NEW,
NUMPY_64,
+ NUMPY_WIN,
] + ARM.builderNames, properties=[]),
] + ARM.schedulers,
@@ -463,7 +476,14 @@
'factory': pypyNumpyCompatability,
'category': 'numpy',
'locks': [TannitCPU.access('counting')],
- },
+ },
+ {'name': NUMPY_WIN,
+ 'slavenames': ["allegro_win32", "SalsaSalsa"],
+ 'builddir': NUMPY_WIN,
+ 'factory': pypyNumpyCompatabilityWin,
+ "locks": [WinSlaveLock.access('counting')],
+ 'category': 'numpy',
+ },
{'name': PYPYBUILDBOT,
'slavenames': ['cobra'],
'builddir': PYPYBUILDBOT,
diff --git a/master/master.cfg b/master/master.cfg
--- a/master/master.cfg
+++ b/master/master.cfg
@@ -23,5 +23,9 @@
if we_are_debugging():
for builderdict in BuildmasterConfig['builders']:
- builderdict["slavenames"] = ['localhost']
+ valid_slaves = ['localhost']
+ for s in builderdict['slavenames']:
+ if s in slaveinfo.passwords:
+ valid_slaves.append(s)
+ builderdict["slavenames"] = valid_slaves
BuildmasterConfig['buildbotURL'] = "http://localhost:%d/" % (httpPortNumber)
More information about the pypy-commit
mailing list