Memory leaks or other things ?

[jurgen.defurne at philips.com]

Hello, list,

I am having the following problem under NT, with Python 2.0 for NT.

For some time I have written a cron.pl project (yes, Perl) with Cygwin, which
runs fine using fork and exec(). Now, to have easier installation of base programs and
script modules, I would like to everything in Python. I had already (re)written much in
Python, but cron.py was the last.

I implemented cron.py using spawn. The complete application is one script.
The main routine syncs on 00 seconds, and then spawns a subprocess, which
reads and processes crontab. For every process that should be run, the script is
respawned with the appropriate arguments, all using Python 2.0 for NT.

The jobs which I have to run allocate a fair amount of memory (160 Mb). The funny
thing is that the jobs which are still running under cron.pl do not leak memory, but the
same job (different arguments) under cron.py does not give its memory back, although
I can't find any processes anymore in the Task Manager.

I am retesting now, with one modification : using sys.exit(0) to terminate the spawned
scripts properly.

Has anyone an idea ? You have the code here. Review at your own will.

Jurgen

#

# External modules
import re
import os
import sys
import string

from time import *
from BitVector import BitVector

# Global variables
year = month = day = hour = min = sec = None
crontab     = 'c:\\etc\\crontab2'
logfile     = 'c:\\var\\log\\cron2.log';
interpreter = 'd:\\python20\\python'
script      = 'cron.py'
shell       = 'c:\\winnt\\system32\\cmd.exe'
fh          = None
initialised = None

# Class definitions
class JobEntry:
    def __init__(self, line = None):
	if line:
	    M, H, d, m, w, c = string.split(line, sep = None, maxsplit = 5)
	    self.mins = self.BitInit(M, 60)
	    self.hour = self.BitInit(H, 24)
	    self.days = self.BitInit(d, 31)
	    self.mnth = self.BitInit(m, 12)
	    self.dows = self.BitInit(w,  7)
	    self.cmnd = string.strip(c)
	else:
	    self.mins = BitVector(60)
	    self.hour = Bitvector(24)
	    self.days = BitVector(31)
	    self.mnth = BitVector(12)
	    self.dows = BitVector(7)
	    self.cmnd = None
    
    def BitInit(self, value, length):
	rv = BitVector(length)

	if value == '*':
	    for i in range(length):
		rv[i] = 1
	else:
	    args = string.split(value, ',')
	    for arg in args:
		if string.find(arg, '-') != -1:
		    start, end = string.split(arg, '-')
		    for i in range(eval(start), eval(end) + 1):
			rv[i] = 1
		else:
		    rv[eval(arg)] = 1
	
	return rv

# Subroutine definitions

def logmsg(msg):
    fh = open(logfile, 'a')
    systime = ctime(time())
    fh.write('%s -- %s\n' % (systime, msg))
    fh.close()

"""
Running a job using spawn is not as easy as using fork and exec. In the normal
Unix a parent process can wait upon the completion of a subprocess, under
Windows this doesn't work. To provide support for this, the script will be
spawned to execute run_job(), and run_job will provide the necessary services
for controlling the job. The job itself will be run synchronously.
"""

def run_job(argv):
    job          = string.join(argv[2:])

    print job

    # Log activity before job is run
    logmsg('Starting : %s' % job)

    # Run job
    os.system(job)

    # Log activity after job has finished
    logmsg('Job %s finished' % job)

    # Exit system in a good fashion
    sys.exit(0)

def read_crontab():
    job_list = []
    fh       = open(crontab, 'r')

    line = fh.readline()
    while line:
	if re.search('^#',   line):
	    line = fh.readline()
	    continue
	if re.search('^ *$', line):
	    line = fh.readline()
	    continue

	job_list.append(JobEntry(line))
	line = fh.readline()

    fh.close()
    return job_list

def process_jobs(list):
    year, mth, day, hrs, min, sec, dow, None, None = localtime(time())

    year  -= 1 
    mth   -= 1
    day   -= 1
    dow   -= 1

    for entry in list:

	if entry.mins[min] == 0: continue
	if entry.hour[hrs] == 0: continue
	if entry.days[day] == 0: continue
	if entry.mnth[mth] == 0: continue
	if entry.dows[dow] == 0: continue
	
	os.spawnl(os.P_NOWAIT, interpreter, 'python', script, 'run', entry.cmnd)

def check_jobs():
    # Lees inhoud van crontab in tabel
    # Scan tabel voor jobs die moeten worden uitgevoerd
    #	Per job :
    #	 	Spawn Python20, python, script, 'run', job en argumenten
    job_list = read_crontab()
    process_jobs(job_list)

    # Exit this script in an orderly fashion
    sys.exit(0)

def start_check_jobs():
    os.spawnl(os.P_NOWAIT, interpreter, 'python', script, 'check')

def sync():
    global initialised
    
    (None, None, None,
     None, None, sec,
     None, None, None) = localtime(time())

    initial = 60 - sec

    if not initialised:
	if initial == 60: initial = 0
	initialised = 1

    sleep(initial)

def main(argv):
    if len(argv) == 1:
	logmsg('cron.py starting up')
	sync()
	while 1:
	    start_check_jobs()
	    sync()
    elif argv[1] == 'check':
	check_jobs()
    elif argv[1] == 'run':
	run_job(argv)

if __name__ == '__main__':
    main(sys.argv)