Sending / receiving WebSocket message over Python socket / WebSocket Client

Question

I wrote a simple WebSocket client. I used the code I found on SO, here: How can I send and receive WebSocket messages on the server side?.

I'm using Python 2.7 and my server is echo.websocket.org on 80 TCP port. Basically, I think that I have a problem with receiving messages. (Or maybe the sending is wrong too?)

At least I am sure that the handshake is all ok, since I receive a good handshake response:

HTTP/1.1 101 Web Socket Protocol Handshake
Access-Control-Allow-Credentials: true
Access-Control-Allow-Headers: content-type
Access-Control-Allow-Headers: authorization
Access-Control-Allow-Headers: x-websocket-extensions
Access-Control-Allow-Headers: x-websocket-version
Access-Control-Allow-Headers: x-websocket-protocol
Access-Control-Allow-Origin: http://example.com
Connection: Upgrade
Date: Tue, 02 May 2017 21:54:31 GMT
Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=
Server: Kaazing Gateway
Upgrade: websocket

And my code:

#!/usr/bin/env python
import socket

def encode_text_msg_websocket(data):
    bytesFormatted = []
    bytesFormatted.append(129)

    bytesRaw = data.encode()
    bytesLength = len(bytesRaw)

    if bytesLength <= 125:
        bytesFormatted.append(bytesLength)
    elif 126 <= bytesLength <= 65535:
        bytesFormatted.append(126)
        bytesFormatted.append((bytesLength >> 8) & 255)
        bytesFormatted.append(bytesLength & 255)
    else:
        bytesFormatted.append(127)
        bytesFormatted.append((bytesLength >> 56) & 255)
        bytesFormatted.append((bytesLength >> 48) & 255)
        bytesFormatted.append((bytesLength >> 40) & 255)
        bytesFormatted.append((bytesLength >> 32) & 255)
        bytesFormatted.append((bytesLength >> 24) & 255)
        bytesFormatted.append((bytesLength >> 16) & 255)
        bytesFormatted.append((bytesLength >> 8) & 255)
        bytesFormatted.append(bytesLength & 255)

    bytesFormatted = bytes(bytesFormatted)
    bytesFormatted = bytesFormatted + bytesRaw
    return bytesFormatted


def dencode_text_msg_websocket(stringStreamIn):
    byteArray = [ord(character) for character in stringStreamIn]
    datalength = byteArray[1] & 127
    indexFirstMask = 2
    if datalength == 126:
        indexFirstMask = 4
    elif datalength == 127:
        indexFirstMask = 10
    masks = [m for m in byteArray[indexFirstMask: indexFirstMask + 4]]
    indexFirstDataByte = indexFirstMask + 4
    decodedChars = []
    i = indexFirstDataByte
    j = 0
    while i < len(byteArray):
        decodedChars.append(chr(byteArray[i] ^ masks[j % 4]))
        i += 1
        j += 1
    return ''.join(decodedChars)

# connect 
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((socket.gethostbyname('echo.websocket.org'), 80))

# handshake
handshake = 'GET / HTTP/1.1\r\nHost: echo.websocket.org\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: gfhjgfhjfj\r\nOrigin: http://example.com\r\nSec-WebSocket-Protocol: echo\r\n' \
        'Sec-WebSocket-Version: 13\r\n\r\n'
sock.send(handshake)
print sock.recv(1024)

# send test msg
msg = encode_text_msg_websocket('hello world!')
sock.sendall(msg)

# receive it back
response = dencode_text_msg_websocket(sock.recv(1024))
print '--%s--' % response

sock.close()

What is wrong here? It gets complicated after the handshake.

The dencode_text_msg_websocket method returns an empty string but it should return the same string I send to the server, which is hello world!.

I DO NOT WANT to use libraries (I know how to use them). The question is about achieving the same thing WITHOUT libraries, using only sockets.

I only want to send a message to echo.websocket.org server and receive a response, that's all. I do not want to modify the headers, just build the headers like they're used by this server. I checked how they should look like using Wireshark, and tried to build the same packets with Python.

For tests below, I used my browser:

Not masked data, from server to client:

Masked data, from client to server:

Answer 1

I have hacked your code into something that at least sends a reply and receives an answer, by changing the encoding to use chr() to insert byte strings rather than decimals to the header. The decoding I have left alone but the other answer here has a solution for that.
The real guts of this is detailed here https://www.rfc-editor.org/rfc/rfc6455.txt
which details exactly what it is that you have to do

#!/usr/bin/env python
import socket
def encode_text_msg_websocket(data):
    bytesFormatted = []
    bytesFormatted.append(chr(129))
    bytesRaw = data.encode()
    bytesLength = len(bytesRaw)
    if bytesLength <= 125:
        bytesFormatted.append(chr(bytesLength))
    elif 126 <= bytesLength <= 65535:
        bytesFormatted.append(chr(126))
        bytesFormatted.append((chr(bytesLength >> 8)) & 255)
        bytesFormatted.append(chr(bytesLength) & 255)
    else:
        bytesFormatted.append(chr(127))
        bytesFormatted.append(chr((bytesLength >> 56)) & 255)
        bytesFormatted.append(chr((bytesLength >> 48)) & 255)
        bytesFormatted.append(chr((bytesLength >> 40)) & 255)
        bytesFormatted.append(chr((bytesLength >> 32)) & 255)
        bytesFormatted.append(chr((bytesLength >> 24)) & 255)
        bytesFormatted.append(chr((bytesLength >> 16)) & 255)
        bytesFormatted.append(chr((bytesLength >> 8)) & 255)
        bytesFormatted.append(chr(bytesLength) & 255)
    send_str = ""
    for i in bytesFormatted:
        send_str+=i
    send_str += bytesRaw
    return send_str

# connect 
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5.0)
try:
    sock.connect((socket.gethostbyname('ws.websocket.org'), 80))
except:
    print "Connection failed"
handshake = '\
GET /echo HTTP/1.1\r\n\
Host: echo.websocket.org\r\n\
Upgrade: websocket\r\n\
Connection: Upgrade\r\n\
Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==\r\n\
Origin: http://example.com\r\n\
WebSocket-Protocol: echo\r\n\
Sec-WebSocket-Version: 13\r\n\r\n\
'
sock.send(bytes(handshake))
data = sock.recv(1024).decode('UTF-8')
print data

# send test msg
msg = encode_text_msg_websocket('Now is the winter of our discontent, made glorious Summer by this son of York')
print "Sent: ",repr(msg)
sock.sendall(bytes(msg))
# receive it back
response = sock.recv(1024)
#decode not sorted so ignore the first 2 bytes
print "\nReceived: ", response[2:].decode()
sock.close()

Result:

HTTP/1.1 101 Web Socket Protocol Handshake
Access-Control-Allow-Credentials: true
Access-Control-Allow-Headers: content-type
Access-Control-Allow-Headers: authorization
Access-Control-Allow-Headers: x-websocket-extensions
Access-Control-Allow-Headers: x-websocket-version
Access-Control-Allow-Headers: x-websocket-protocol
Access-Control-Allow-Origin: http://example.com
Connection: Upgrade
Date: Mon, 08 May 2017 15:08:33 GMT
Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk=
Server: Kaazing Gateway
Upgrade: websocket


Sent:  '\x81MNow is the winter of our discontent, made glorious Summer by this son of York'

Received:  Now is the winter of our discontent, made glorious Summer by this son of York

I should note here, that this is going to be a pig to code without pulling in some extra libraries, as @gushitong has done.

Answer 2

Accoding to https://www.rfc-editor.org/rfc/rfc6455#section-5.1:

You should mask the client frames. (And the server frames is not masked at all.)

• a client MUST mask all frames that it sends to the server (see Section 5.3 for further details). (Note that masking is done whether or not the WebSocket Protocol is running over TLS.) The server MUST close the connection upon receiving a frame that is not masked. In this case, a server MAY send a Close frame with a status code of 1002 (protocol error) as defined in Section 7.4.1. A server MUST NOT mask any frames that it sends to the client. A client MUST close a connection if it detects a masked frame。

---

This is a working version:

import os
import array
import six
import socket
import struct

OPCODE_TEXT = 0x1

try:
    # If wsaccel is available we use compiled routines to mask data.
    from wsaccel.xormask import XorMaskerSimple
    
    def _mask(_m, _d):
        return XorMaskerSimple(_m).process(_d)

except ImportError:
    # wsaccel is not available, we rely on python implementations.
    def _mask(_m, _d):
        for i in range(len(_d)):
            _d[i] ^= _m[i % 4]

        if six.PY3:
            return _d.tobytes()
        else:
            return _d.tostring()


def get_masked(data):
    mask_key = os.urandom(4)
    if data is None:
        data = ""

    bin_mask_key = mask_key
    if isinstance(mask_key, six.text_type):
        bin_mask_key = six.b(mask_key)

    if isinstance(data, six.text_type):
        data = six.b(data)

    _m = array.array("B", bin_mask_key)
    _d = array.array("B", data)
    s = _mask(_m, _d)

    if isinstance(mask_key, six.text_type):
        mask_key = mask_key.encode('utf-8')
    return mask_key + s


def ws_encode(data="", opcode=OPCODE_TEXT, mask=1):
    if opcode == OPCODE_TEXT and isinstance(data, six.text_type):
        data = data.encode('utf-8')

    length = len(data)
    fin, rsv1, rsv2, rsv3, opcode = 1, 0, 0, 0, opcode

    frame_header = chr(fin << 7 | rsv1 << 6 | rsv2 << 5 | rsv3 << 4 | opcode)

    if length < 0x7e:
        frame_header += chr(mask << 7 | length)
        frame_header = six.b(frame_header)
    elif length < 1 << 16:
        frame_header += chr(mask << 7 | 0x7e)
        frame_header = six.b(frame_header)
        frame_header += struct.pack("!H", length)
    else:
        frame_header += chr(mask << 7 | 0x7f)
        frame_header = six.b(frame_header)
        frame_header += struct.pack("!Q", length)

    if not mask:
        return frame_header + data
    return frame_header + get_masked(data)


def ws_decode(data):
    """
    ws frame decode.
    :param data:
    :return:
    """
    _data = [ord(character) for character in data]
    length = _data[1] & 127
    index = 2
    if length < 126:
        index = 2
    if length == 126:
        index = 4
    elif length == 127:
        index = 10
    return array.array('B', _data[index:]).tostring()


# connect
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((socket.gethostbyname('echo.websocket.org'), 80))

# handshake
handshake = 'GET / HTTP/1.1\r\nHost: echo.websocket.org\r\nUpgrade: websocket\r\nConnection: ' \
            'Upgrade\r\nSec-WebSocket-Key: gfhjgfhjfj\r\nOrigin: http://example.com\r\nSec-WebSocket-Protocol: ' \
            'echo\r\n' \
            'Sec-WebSocket-Version: 13\r\n\r\n'

sock.send(handshake)
print(sock.recv(1024))

sock.sendall(ws_encode(data='Hello, China!', opcode=OPCODE_TEXT))

# receive it back
response = ws_decode(sock.recv(1024))
print('--%s--' % response)

sock.close()