Ordering tests in a testsuite

[Peter Otten]

Ulrich Eckhardt wrote:

> I'm currently working on a testsuite using Python's unittest library. This
> works all good and fine, but there's one thing where I haven't seen an
> elegant solution to yet, and that is the ordering. Currently, it takes all
> classes and orders them alphabetically and then takes all test functions
> therein and runs those alphabetically, too. However, sometimes it doesn't
> make sense to run test_bar() if test_foo() already failed, because they
> basically build upon each other. However, test_bar() is still run first,
> and test_foo() second.
> 
> What I sometimes do is to simply number them, like test_1_foo() and
> test_2_bar(), but that seems ugly. I'm not even looking for a way to
> express complicated relations between tests, but is there a less ugly way
> to explicitly order them?

You can pass unittest.main() a custom test loader which in turn can define a 
custom sortTestMethodsUsing() method. Here's a fancy example:

import unittest

def _toposort(data):
    # adapted from http://code.activestate.com/recipes/577413-topological-
sort/
    for k, v in data.items():
        v.discard(k) # Ignore self dependencies
    extra_items_in_deps = reduce(set.union, data.values()) - 
set(data.keys())
    data.update((item, set()) for item in extra_items_in_deps)
    while True:
        ordered = set(item for item,dep in data.items() if not dep)
        if not ordered:
            break
        for item in sorted(ordered):
            yield item
        data = dict((item, (dep - ordered)) for item,dep in data.items()
                if item not in ordered)

def _get_name(func):
    try:
        return func.__name__
    except AttributeError:
        pass
    assert isinstance(func, str)
    return func

class Cmp(object):
    def __init__(self):
        self._deps = {}
    def depends(self, f, names):
        k = _get_name(f)
        assert k not in self._deps
        self._deps[k] =  set(_get_name(n) for n in names)
        return f
    def make_loader(self):
        lookup = dict((name, index) for index, name
                      in enumerate(_toposort(self._deps)))
        def compare(a, b):
            return cmp(lookup[a], lookup[b])
        class TestLoader(unittest.TestLoader):
            pass
        loader = TestLoader()
        loader.sortTestMethodsUsing = compare
        return loader
    def depends_on(self, *names):
        def d(f):
            return self.depends(f, names)
        return d


c = Cmp()
depends_on = c.depends_on

class A(unittest.TestCase):
    @depends_on("test_beta")
    def test_alpha(self):
        pass
    @depends_on("test_gamma")
    def test_beta(self):
        pass
    def test_gamma(self):
        pass
    @depends_on(test_gamma)
    def test_delta(self):
        pass
    @depends_on(test_alpha, test_beta)
    def test_epsilon(self):
        pass

if __name__ == "__main__":
    unittest.main(testLoader=c.make_loader())

Peter