Python threading module error: 'NoneType' object is not callable

Question

My screen displays <type 'exceptions.TypeError'>: 'NoneType' object is not callable when I run this code below:

import threading
import urllib2
import Queue
import time

hosts = ["http://baidu.com", "http://yahoo.com"]

queue = Queue.Queue()

class ThreadUrl(threading.Thread):
    """ Threaded Url Grab """
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            #grabs host from queue
            host = self.queue.get()

            #grabs urls of hosts and prints first 1024 byte of page
            url = urllib2.urlopen(host)

            #signals to queue job is done
            self.queue.task_done()

start = time.time()
def main():
    for i in range(2):
        t = ThreadUrl(queue)
        t.setDaemon(True)
        t.start()

        for host in hosts:
            queue.put(host)

    queue.join()

main()
print "Elapsed Time: %s" % (time.time() - start)

This is error details:

Exception in thread Thread-3 (most likely raised during interpreter shutdown):
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 808, in __bootstrap_inner
File "url_thread.py", line 21, in run
File "/usr/local/lib/python2.7/Queue.py", line 168, in get
File "/usr/local/lib/python2.7/threading.py", line 332, in wait
<type 'exceptions.TypeError'>: 'NoneType' object is not callable

What's wrong with my code? Thanks a lot.

Answer 1

It's a python 2.7 bug - Shutdown exception in daemon thread. You could make the threads non-deamon and pass a sentinel through the queue to signal them to exit, then join them from the main thread.

import threading
import urllib2
import Queue
import time

hosts = ["http://baidu.com", "http://yahoo.com"]

queue = Queue.Queue()

class ThreadUrl(threading.Thread):
    """ Threaded Url Grab """
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            #grabs host from queue
            host = self.queue.get()
            if host is None:
                self.queue.task_done()
                return

            #grabs urls of hosts and prints first 1024 byte of page
            url = urllib2.urlopen(host)

            #signals to queue job is done
            self.queue.task_done()

start = time.time()
def main():
    threads = [ThreadUrl(queue) for _ in range(2)]
    map(lambda t: t.start() threads)
    for i in range(2):
        for host in hosts:
            queue.put(host)
    for t in threads:
        queue.put(None)
    queue.join()
    map(lambda t: t.join() threads)

main()
print "Elapsed Time: %s" % (time.time() - start)