Python - How to wait on both queue and a socket on same time

Question

I want to read messages from either a Queue.Queue or a TCP socket, whichever comes first. How can it be achieved without resorting to 2 threads ? Platform is CPython 2.7.5 on Windows

Answer 1

There is a very nice trick to do this here that applies to your problem.

import queue
import socket
import os

class PollableQueue(queue.Queue):
    def __init__(self):
        super().__init__()
        # Create a pair of connected sockets
        if os.name == 'posix':
            self._putsocket, self._getsocket = socket.socketpair()
        else:
            # Compatibility on non-POSIX systems
            server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            server.bind(('127.0.0.1', 0))
            server.listen(1)
            self._putsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._putsocket.connect(server.getsockname())
            self._getsocket, _ = server.accept()
            server.close()

    def fileno(self):
        return self._getsocket.fileno()

    def put(self, item):
        super().put(item)
        self._putsocket.send(b'x')

    def get(self):
        self._getsocket.recv(1)
        return super().get()

Answer 2

To do it in a single thread, you'll have to use non-blocking methods, and merge them into a single event loop. I'm actually using select instead of non-blocking socket I/O here, since it's slightly cleaner if you need to read from multiple sockets...

import socket
import select
import Queue
import time

TIMEOUT = 0.1   # 100ms


def process_queue_item(item):
    print 'Got queue item: %r' % item


def process_socket_data(data):
    print 'Got socket data: %r' % data


def main():

    # Build queue
    queue = Queue.Queue()
    for i in range(10):
        queue.put(i)
    queue.put(None)   # Using None to indicate no more data on queue
    queue_active = True

    # Build socket
    sock = socket.socket()
    sock.connect(('www.google.com', 80))
    sock.send('GET / HTTP/1.0\r\n\r\n')
    socket_active = True

    # Main event loop
    while 1:

        # If there's nothing to read, bail out
        if not (socket_active or queue_active):
            break

        # By default, sleep at the end of the loop
        do_sleep = True

        # Get data from socket without blocking if possible
        if socket_active:
            r, w, x = select.select([sock], [], [], TIMEOUT)
            if r:
                data = sock.recv(64)
                if not data:    # Hit EOF
                    socket_active = False
                else:
                    do_sleep = False
                    process_socket_data(data)

        # Get item from queue without blocking if possible
        if queue_active:
            try:
                item = queue.get_nowait()
                if item is None:  # Hit end of queue
                    queue_active = False
                else:
                    do_sleep = False
                    process_queue_item(item)
            except Queue.Empty:
                pass

        # If we didn't get anything on this loop, sleep for a bit so we
        # don't max out CPU time
        if do_sleep:
            time.sleep(TIMEOUT)


if __name__ == '__main__':
    main()

Output looks like...

Got socket data: 'HTTP/1.0 302 Found\r\nLocation: http://www.google.co.uk/\r\nCache-Co'
Got queue item: 0
Got socket data: 'ntrol: private\r\nContent-Type: text/html; charset=UTF-8\r\nSet-Cook'
Got queue item: 1
Got socket data: 'ie: PREF=ID=a192ab09b4c13176:FF=0:TM=1373055330:LM=1373055330:S='
Got queue item: 2
etc.