Running a python farm

[Brian Kelley]

Ian McConnell wrote:
> What's the pythonic way of sending out a set of requests in parallel?
> 
> My program throws an image at the server and then waits for the result. I'm
> currently using this bit of socket code to send an image to server on
> another machine.
> 
> 
> import socket
> import SocketServer
> import cPickle
> 
> def oneclient(host, port, array):
>     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
>     sock.connect((host, port))
>     sent = sock.sendall(cPickle.dumps(array, 0))
>     sock.shutdown(1)
>     rfile = sock.makefile('rb')
>     return cPickle.loads(rfile.read())
> 

You don't need threads here.  We can leverage the fact that 
select.select can poll to see if results are ready, how about something 
like:

class Client:
       def __init__(host, port, array):
           sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
           self.sock = sock
           self.host = host
           self.port = port
           sock.connect((host, port))
           sent = sock.sendall(cPickle.dumps(array, 0))
           sock.shutdown(1)
           self.rfile = sock.makefile('rb')

        def result(self):
           r,w,e = selelect.select([self.rfile], [], [], 1)
           if not r:
              return None
           else:
              return cPickle.loads(self.rfile.read())

all_images = [...]
hosts = [(host,port), (host1,port1)]
client_list = []
# assign images to hosts
for host, port in hosts:
     client_list.append(Client(host, port, all_images.pop()))

# process
while client_list:
       next_client_list = []
       for o in client_list:
           res = o.result()
           if o is not None:
              # do something with image result
              if all_images: # if we have images assign to completed host
                 next_client_list.append(Client(o.host, o.port,
                                         all_images.pop())
           else:
               next_client_list.append(o)
       client_list = next_client_list