multiprocessing shows no benefit

Jason jasonhihn at gmail.com
Wed Oct 18 11:46:35 EDT 2017


#When I change line19 to True to use the multiprocessing stuff it all slows down. 

from multiprocessing import Process, Manager, Pool, cpu_count
from timeit import default_timer as timer

def f(a,b):
	return dict_words[a]-b

def f_unpack(args):
	return f(*args)

def init():
	global dict_words, dict_keys
	d = {str(k):k for k in range(100000)}
	for k in d:
		dict_words[k] = d[k]
		dict_keys.append(k)

if __name__ == '__main__':
	print 'CPUs:', cpu_count() # 4 on my machine
	if False: # make this a zero to use PODs
		'''
		CPUs: 4
		pool.map: 47.1494848728 1000000 21209.1394571
		map: 49.2051260471 1000000 20323.0858314
		'''
		manager = Manager()
		dict_words = manager.dict()
		dict_keys = manager.list()
	else:
		'''
		CPUs: 4
		pool.map: 3.2546248436 1000000 307255.074872
		map: 1.19043779373 1000000 840027.093617
		'''
		dict_words = {}
		dict_keys = []

	init()

	pool = Pool(cpu_count())
	z=5
	funcs = {'pool.map:': pool.map, 'map': map}
	for name in funcs:
		start = timer()
		x = funcs[name](f_unpack, [(k,z) for k in dict_keys])
		duration = float(timer() -start)
		print funcs[name], duration, len(x), len(x) / duration



More information about the Python-list mailing list