thread help

[Bart Nessux]

Scott David Daniels wrote:
> Aahz wrote:
> 
>> Bart Nessux  <bart_nessux at hotmail.com> wrote:
>>
>>> Could someone show me how I can make this work correctly? I want to 
>>> probe 
> 
>  >>64 unique IP address for HTTP servers simultaneously, ...
> 
>> Create a threading.Thread subclass that takes one IP address and a list
>> of ports to scan.  Start 64 instances of this class, each with a
>> different IP address.
> 
> 
> An alternative is to create a que into which you push IP addresses to
> contact, and have each thread read addresses off the queue when they are
> free to process them.  This has the advantage of decoupling the number
> of threads from the number of addresses you want to examine.
> 
> -Scott David Daniels
> Scott.Daniels at Acm.Org

I like this idea. I read up on the queue and threading module at 
python.org and a few other sites around the Web and came up with this, 
however, it doesn't work. I get these errors when it runs:

Exception in thread Thread-149:
Traceback (most recent call last):
   File "/usr/lib/python2.3/threading.py", line 434, in __bootstrap
     self.run()
   File "/usr/lib/python2.3/threading.py", line 414, in run
     self.__target(*self.__args, **self.__kwargs)
   File "www_reads_threaded_1.py", line 49, in sub_thread_proc
     f = urllib2.urlopen(url).read()
   File "/usr/lib/python2.3/urllib2.py", line 129, in urlopen
     return _opener.open(url, data)
   File "/usr/lib/python2.3/urllib2.py", line 324, in open
     type_ = req.get_type()
AttributeError: 'NoneType' object has no attribute 'get_type'

The problem I have is this: I know too little about thread programming. 
If anyone thinks the code I have below could be made to work for my 
tasks (probe 65,000 IPs for HTTP servers using threads to speed things 
up), then please *show* me how I might change it in order for it to work.

Thanks again,
Bart

networks = []
hosts = []
urls = []
#socket.setdefaulttimeout(30)
max_threads = 2
http_timeout = 30
start_time = time.time()

# Add the network 192.168.0 possibility.
networks.append("192.168.0.")

# Generate and add networks 192.168.1-255 to the list of networks.
n = 0
while n < 255:
    n = n + 1
    networks.append("192.168.%s." %(n))

# Generate and add hosts 1-255 to each network
for network in networks:
    h = 0
    # Add the n.n.n.0 host possibility
    hosts.append(network+str(h))
    while h < 255:
       h = h + 1
       hosts.append(network+str(h))

for ip in hosts:
    ip = "http://" + ip
    urls.append(ip)

urls = dict(zip(urls,urls))
# print urls

# Create a queue of urls to feed the threads
url_queue = Queue.Queue()
for url in urls:
     url_queue.put(url)
#     print url

def test_HTTP(url_queue):
    def sub_thread_proc(url, result):
#       try:
       f = urllib2.urlopen(url).read()
#       except Exception:
#          print "Exception"
#       else:
       result.append(url)
    while 1:
       try:
          url = url_queue.get(0)
       except Queue.Empty:
          return
       result = []
       sub_thread = threading.Thread(target=sub_thread_proc, 
args=(url,result))
       sub_thread.setDaemon(True)
       sub_thread.start()
       sub_thread.join(http_timeout)
    print result

test_HTTP(urls)