subprocess.Popen(cmd) question

[Peter Otten]

WolfgangZ wrote:

> I'm starting some subprocesses inside a loop. The processes run
> independent and dont need any communication between each other. Due to
> memory issues I need to limit the number of running processes to around
> 10. How can I insert a break into my loop to wait until some processes
> are finished?
> 
> Some minimal examplecode:
> 
> import subprocess
> for i in range(0,100):
>      cmd='ping localhost'
>      p=subprocess.Popen(cmd)
>      p.wait()
> 

Just polling the processes may be good enough:

import random
import subprocess
import sys
import time
from itertools import islice
from functools import partial

PROCESSES = 100
SIMULTANEOUS = 10

def info(*args):
    print >> sys.stderr, " ".join(str(a) for a in args)

class Process(object):
    def __init__(self, index):
        self.index = index
        info("starting process #%d" % index)
        self.process = subprocess.Popen(["ping", "localhost", "-w", "%d" %
random.randrange(1, 6)])
    def __nonzero__(self):
        running = self.process.poll() is None
        if not running:
            # XXX ugly side effect
            info("process #%d terminated" % self.index)
        return running

def processes():
    for i in range(PROCESSES):
        yield partial(Process, i)

def main():
    starters = processes()
    running = [sp() for sp in islice(starters, SIMULTANEOUS)]
    while running:
        before = len(running)
        running = [p for p in running if p]
        after = len(running)
        if before == after:
            info("waiting")
            time.sleep(1)
        else:
            running.extend(sp() for sp in islice(starters, SIMULTANEOUS -
len(running)))
    info("that's all, folks")

if __name__ == "__main__":
    main()

Peter