process do not join?

[xeon Mailinglist]

This log came when I launched the command:

python -m trace --trace myclass.py


On Saturday, April 5, 2014 3:18:34 AM UTC+1, xeon Mailinglist wrote:
> I am trying to debug my program that launch processes to run a function to copy data between hosts located really far away from each other. The end of my function are in the  orders.py and mergedirs.py.
> 
> 
> 
> From this point onwards, in is python code. The problem is that this code hangs in the last instruction "forking()", instead of my program continue.
> 
> 
> 
> The call of my function were made with: outputs = parmap(mergeDirs, args). This is my parmap function.
> 
> 
> 
> It seems that the function mergeDirs ends, but the process does not join. I checked that the mergeDirs is working properly, because I ran the code without processes, and it works fine.
> 
> 
> 
> I don't know what is happening. How can I debug this problem?
> 
> 
> 
> def parmap(f, X):
> 
>     pipe = [Pipe() for x in X]
> 
>     # 2 - what is happening with the tuples (c,x) and (p, c)?
> 
>     proc = [Process(target=spawn(f), args=(c, x))
> 
>             for x, (p, c) in zip(X, pipe)]
> 
>     #[p.start() for p in proc]
> 
>     for p in proc:
> 
>         print("Spawn")
> 
>         p.start()
> 
>     #[p.join() for p in proc]
> 
>     for p in proc:
> 
>         print("Joining")
> 
>         p.join()
> 
>     return [p.recv() for (p, c) in pipe]
> 
> 
> 
> 
> 
> 
> 
> 
> 
> Copy data time: 104.863273859
> 
> orders.py(99):     return cluster
> 
> mergedirs.py(48):     return (new_included, command, poutput)
> 
> spawn.py(7):         pipe.close()
> 
> process.py(259):                 exitcode = 0
> 
> process.py(261):                 util._exit_function()
> 
>  --- modulename: util, funcname: _exit_function
> 
> util.py(303):     info('process shutting down')
> 
>  --- modulename: util, funcname: info
> 
> util.py(77):     if _logger:
> 
> util.py(304):     debug('running all "atexit" finalizers with priority >= 0')
> 
>  --- modulename: util, funcname: debug
> 
> util.py(73):     if _logger:
> 
> util.py(305):     _run_finalizers(0)
> 
>  --- modulename: util, funcname: _run_finalizers
> 
> util.py(257):     if _finalizer_registry is None:
> 
> util.py(263):     if minpriority is None:
> 
> util.py(266):         f = lambda p : p[0][0] is not None and p[0][0] >= minpriority
> 
> util.py(268):     items = [x for x in _finalizer_registry.items() if f(x)]
> 
> util.py(269):     items.sort(reverse=True)
> 
> util.py(271):     for key, finalizer in items:
> 
> util.py(279):     if minpriority is None:
> 
> util.py(307):     if current_process() is not None:
> 
>  --- modulename: process, funcname: current_process
> 
> process.py(63):     return _current_process
> 
> util.py(318):         for p in active_children():
> 
>  --- modulename: process, funcname: active_children
> 
> process.py(69):     _cleanup()
> 
>  --- modulename: process, funcname: _cleanup
> 
> process.py(78):     for p in list(_current_process._children):
> 
> process.py(70):     return list(_current_process._children)
> 
> util.py(323):         for p in active_children():
> 
>  --- modulename: process, funcname: active_children
> 
> process.py(69):     _cleanup()
> 
>  --- modulename: process, funcname: _cleanup
> 
> process.py(78):     for p in list(_current_process._children):
> 
> process.py(70):     return list(_current_process._children)
> 
> util.py(327):     debug('running the remaining "atexit" finalizers')
> 
>  --- modulename: util, funcname: debug
> 
> util.py(73):     if _logger:
> 
> util.py(328):     _run_finalizers()
> 
>  --- modulename: util, funcname: _run_finalizers
> 
> util.py(257):     if _finalizer_registry is None:
> 
> util.py(263):     if minpriority is None:
> 
> util.py(264):         f = lambda p : p[0][0] is not None
> 
> util.py(268):     items = [x for x in _finalizer_registry.items() if f(x)]
> 
> util.py(269):     items.sort(reverse=True)
> 
> util.py(271):     for key, finalizer in items:
> 
> util.py(279):     if minpriority is None:
> 
> util.py(280):         _finalizer_registry.clear()
> 
> process.py(278):         util.info('process exiting with exitcode %d' % exitcode)
> 
>  --- modulename: util, funcname: info
> 
> util.py(77):     if _logger:
> 
> process.py(279):         return exitcode
> 
> forking.py(127):                 sys.stdout.flush()
> 
> forking.py(128):                 sys.stderr.flush()
> 
> forking.py(129):                 os._exit(code)