Extreme Python Testing
John Dell'Aquila
dellaq2 at pdq.net
Tue Jun 8 00:20:14 EDT 1999
I admire the Smalltalk "Extreme Programming" methodology enough that
I'm applying it in my Python development. XP is impossible without an
automated test framework, so the first thing I did was write a Python
version of Kent Beck's Testing Framework.
Shortly thereafter, I saw Tim Peter's brilliant doctest.py and
realized there was no need to write separate TestCase classes. With
doctest, the test case coulde *be* the documentation, inseparably
linked to the object being exercised. After a lot of cutting and
twisting, a very simple framework emerged.
I like the result well enough that I sometimes catch myself doing the
Right Thing - writing test cases *before* starting to code. I attach
xp.py in the hope that others may find it useful and I offer apologies
in advance if my abuse of docStrings treads on *your* abuse.
John
# Module xp
# John Dell'Aquila <jbd at alum.mit.edu>
# 6 June 1999
# Public domain
# Usual disclaimers apply: "as is", "use at your own risk",
# "no warranty of fitness or merchantability", . . .
""" Module xp - Extreme Python Test Framework.
Inspired by Kent Beck's Testing Framework (see Extreme Programming web
pages at http://c2.com/cgi/wiki?ExtremeProgrammingRoadmap) and derived
from Tim Peter's doctest.py (comp.lang.python archives for March,
1999).
xp.Test examines ALL docStrings in specified modules, classes, methods
or functions and *executes* any lines having the Python interpreter
prompt (>>>) as their first non-whitespace. DocStrings are classified
as passed (no errors), failed (exception raised) or noTest (no
docString or no code within docString).
The general idea is to create test objects and then verify their
correct behavior with 'assert' statements. Multi-line Python
statements are explicitly NOT supported; complex tests must be
implemented as helper functions/methods in the code being
tested. Exception testing IS supported (see TestResult.runDocString)
and so are unqualified imports ('from foo import *' will NOT cause
foo's objects to be tested).
See the docStrings in this file for examples.
Usage:
import xp
tst = xp.Test() # create Test harness
tst.check(myModule) # check some stuff
tst.check(moduleX.Class1).check(moduleX.Class2)
. . .
print tst.result # print summary
print tst.result.errors() # display formatted error messages
print tst.result.passed # list objects that passed tests
print tst.result.noTest # list objects that had no tests
print tst.result.failed # list raw error tuples
Sample test summary:
<TestResult: 10 passed, 1 failed, 0 noTest, 0.03 sec>
Sample error message:
-------------------------------------------------
DocString: FUNCTION:__main__.foo
Statement: assert 1==2
Exception: AssertionError:
>>> None # stops noTest snivels, i.e this space intentionally blank
"""
__version__ = '1.0'
import sys, types, time, re, string, traceback
_PS1 = re.compile(r'\s*>>>\s*').match
_summaryFmt = (
"<TestResult: %(passed)d passed, %(failed)d failed, "
"%(noTest)d noTest, %(elapsed)g sec>"
)
_errorFmt = (
"-------------------------------------------------\n"
"DocString: %s\n"
"Statement: %s\n"
"Exception: %s\n"
)
_Module = types.ModuleType
_Class = types.ClassType
_Method = types.MethodType
_Func = types.FunctionType
_Dict = types.DictionaryType
_None = types.NoneType
_Long = types.LongType
_BuiltIn = types.BuiltinFunctionType
_String = types.StringType
_typeAbbrev = {
_Method: 'METHOD',
_Long: 'LONG',
_BuiltIn: 'BUILTIN'
}
_expect_Fn = """
def _expect_(stmt, exc):
try:
exec stmt
except exc:
return
else:
raise '_expect_', '%s not raised' % exc.__name__
"""
def moduleName(obj):
""" Return object's module name, if known, otherwise None.
>>> assert moduleName(re) == 're'
>>> assert moduleName(re.compile) == 're'
>>> assert moduleName(re.RegexObject) == 're'
>>> assert moduleName(re.RegexObject.match) == 're'
>>> assert moduleName('foo') == None
"""
if type(obj) == _Module:
return obj.__name__
if type(obj) == _Class:
return obj.__module__
if type(obj) == _Method:
return obj.im_class.__module__
if type(obj) == _Func:
return obj.func_globals['__name__']
return None
def objRepr(obj):
""" Return printable representation of object.
>>> assert objRepr(re) == 'MODULE:re'
>>> assert objRepr(re.RegexObject) == 'CLASS:re.RegexObject'
>>> assert objRepr(Test.check) == 'METHOD:%s.Test.check' % __name__
>>> assert objRepr(objRepr) == 'FUNCTION:%s.objRepr' % __name__
>>> assert objRepr('foo') == 'STRING:'
>>> assert objRepr(3L) == 'LONG:'
>>> assert objRepr(string.lower) == 'BUILTIN:lower'
"""
typ = _typeAbbrev.get( type(obj),
string.upper(type(obj).__name__) )
if type(obj) == _Module:
return '%s:%s' % (typ, obj.__name__)
if type(obj) == _Class:
return '%s:%s.%s' % (typ, obj.__module__, obj.__name__)
if type(obj) == _Method:
return '%s:%s.%s' % (typ, str(obj.im_class), obj.__name__)
if type(obj) == _Func:
return '%s:%s.%s' % (typ, obj.func_globals['__name__'],
obj.__name__)
if hasattr(obj, '__name__'):
return '%s:%s' % (typ, obj.__name__)
return '%s:' % typ
class TestResult:
""" Accumulate test results. Helper class to Test.
>>> None
"""
def __init__(self):
""" >>> None """
self.start = self.stop = 0
self.passed = []
self.failed = []
self.noTest = []
def runDocString(self, obj, env):
""" Execute obj.__doc__ in a *shallow* copy of the
environment dictionary, env. If possible, insert
_expect_ function into environment to allow exception
testing. Tally results and timestamp.
>>> tr = TestResult() # check for *expected* exception
>>> _expect_('tr.runDocString(3, {})', AssertionError)
"""
assert type(obj) in (_Module, _Class, _Method, _Func)
if not self.start:
self.start = time.time()
name = objRepr(obj)
globs = env.copy()
if not globs.has_key('_expect_'):
exec _expect_Fn in globs
status = None
for line in string.split(obj.__doc__ or '', '\n'):
matchPS1 = _PS1(line)
stmt = matchPS1 and line[matchPS1.end(0):]
if not stmt:
continue
try:
exec stmt in globs
status = 1
except:
etype, evalue = sys.exc_info()[:2]
edesc = traceback.format_exception_only(etype, evalue)
edesc = string.strip(string.join(edesc, ''))
status = 0
break
if status == None:
self.noTest.append(name)
elif status == 0:
self.failed.append((name, stmt, edesc))
elif status == 1:
self.passed.append(name)
self.stop = time.time()
def __repr__(self):
""" Display summary results.
>>> None
"""
vars = { 'passed': len(self.passed),
'failed': len(self.failed),
'noTest': len(self.noTest),
'start': time.ctime(self.start),
'stop': time.ctime(self.stop),
'elapsed' : self.stop-self.start }
return _summaryFmt % vars
def errors(self):
""" Return error details as printable string.
>>> None
"""
return string.join(map(lambda x, f=_errorFmt: f%x, self.failed),
'')
class Test:
""" DocString tester.
>>> None
"""
def __init__(self):
""" >>> None """
self.result = TestResult()
def check(self, obj, env=None):
""" Recursively test docStrings of object and any subobjects
(module or class members). Import safe - does not cross module
boundaries (use successive calls to check other modules).
For convenience, Strings are taken as module names, i.e.
check(__name__) tests the current module.
DocStrings execute in env dictionary, if supplied, otherwise
in the appropriate module namespace. A *shallow* copy
is made for each docString, so temporary variables may be
used freely, but changing mutable objects can 'pollute'
environment of succeeding tests.
>>> None
"""
assert type(obj) in (_Module, _Class, _Method, _Func, _String)
if type(obj) == _String:
thisModuleName, obj = obj, sys.modules[obj]
else:
thisModuleName = moduleName(obj)
if not env:
env = sys.modules[thisModuleName].__dict__
self.result.runDocString(obj, env)
if type(obj) not in (_Module, _Class):
return self
for name in obj.__dict__.keys():
nxtObj = getattr(obj, name) # NOT obj.__dict__[name] !!!
if type(nxtObj) not in (_Class, _Method, _Func):
continue
if moduleName(nxtObj) == thisModuleName:
self.check(nxtObj, env)
return self
if __name__ == '__main__':
tst = Test().check('__main__')
print tst.result
print tst.result.errors()
More information about the Python-list
mailing list