Memory leaks in Python 1.6

[Charles G Waldman]

I've been running a small script I wrote called "leaktest" (attached
below) which runs tests from the the Python/Lib/test directory in a
loop, while watching the memory use of the process.  I've found quite
a few memory leaks this way, and am working on fixing them.  I will be
submitting a bunch of patches to the patches list over the next
several days.




Script: leaktest.py

#!/usr/bin/env python

"""
Runs tests repeatedly, looking for memory leaks
If you have gnuplot installed you can also get nice plots of memory
usage vs. time

Usage:  leaktest [-max max] [-min min] [-check check] [testname] [testname]...

runs one or more tests repeatedly, looking for memory leaks
The tests can be specified as module name (test_gzip) or file name
(/path/to/test_gzip.py)

memory usage is checked every "check" iterations of the test.

at least "min" readings are taken, and at most "max" readings.  If no
memory leak is detected after "min", the test stops.

If "DO_LOG" is set to 1, memory usage will be logged to a file.
If "DO_PLOT" is set to 1, pretty pictures will be drawn.
If "VERBOSE" is set to 1, memory usage will be printed during the test
"""


import sys
import os
import time
import string

DO_LOG=1
DO_PLOT=1
VERBOSE=0

def open_file(name):
    if os.path.exists(name):
        os.system("cp -b %s %s.bak"%(name,name))
    f=open(name,'w')
    return f

def mem_usage(pid):
    p=os.popen('ps uwp %s'%pid)
    lines=p.readlines()
    status=p.close()
    if status or len(lines)!=2:
        return None
    return int(string.split(lines[1])[4])

def no_increase(l):
    for i in xrange(1,len(l)):
        if l[i]>l[i-1]:
            return 0
    return 1
        
def run_test(testname, min_readings=10, max_readings=50, check_interval=10):
    print "RUN %s min %d max %d check %d"%(testname,min_readings,
                                           max_readings, check_interval)
    if DO_LOG:
        log = open_file('%s.log'%testname)
    logdata=[]
    ret=1
    pid=os.getpid()

    stdout, stderr = sys.stdout, sys.stderr
    sys.stdout, sys.stderr = open_file('%s.out'%testname), open_file('%s.err'%testname)
    
    for loop_readings in xrange(max_readings):
        mem = mem_usage(pid)
        if not mem:
            break
        if VERBOSE:
            stdout.write('%s\n'%mem)
        logdata.append(mem)
        if DO_LOG:
            log.write('%s\n'%mem)
            log.flush()

        for n in xrange(check_interval):
            try:
                if loop_readings==0:
                    exec "import test.%s"%testname
                else:
                    exec('reload(test.%s)'%testname)
            except:
                ret=-1
                break

        if ret<0:
            break

        if len(logdata)>=min_readings and no_increase(logdata[-min_readings:]):
            ret=0
            break

    sys.stdout.close()
    sys.stderr.close()
    sys.stdout, sys.stderr = stdout, stderr
    
    if DO_LOG:
        log.close()

    return ret, logdata
            
def do_plot(testname):
    cmdfile=open('%s.gnuplot' % testname, 'w')
    template='set nokey\nset xlabel "Iterations"\nset ylabel "Memory Use (KB)"\n\
                              set title "%s\nplot "%s" with lines\npause 60\n'
    cmdfile.write(template % (testname, testname+'.log'))
    cmdfile.close()
    os.system("gnuplot %s.gnuplot &"%testname)
    
def main(args):
    check_interval=10
    min_readings=10
    max_readings=50
    while args:
        arg = args.pop(0)
        if arg=='-min':
            min_readings = int(args.pop(0))
            continue
        if arg=='-max':
            max_readings = int(args.pop(0))
            continue
        if arg=='-check':
            check_interval = int(args.pop(0))
            continue
        ## allow the test module names to be specified as full pathnames
        ## for ease in calling this from shellscripts
        arg = os.path.split(arg)[1]
        arg = os.path.splitext(arg)[0]
        status, data=run_test(arg, min_readings, max_readings, check_interval)
        if status<0:
            print "test", arg, "did not run"
        elif status==0:
            print arg, "no memory leak detected", data
        else:
            print arg, "leaks memory", data
            if DO_PLOT:
                do_plot(arg)
    
if __name__ == '__main__':
    main(sys.argv[1:])