Create http request using TcpClient

Question

I have created an empty asp.net web application where I have a simple aspx page:

protected void Page_Load(object sender, EventArgs e)
{
    Response.Write("Hello world");
}

when i goto http://localhost:2006/1.aspx I see a page that says "Hello world".

---

Ok so on c# if I do:

WebClient webClient = new WebClient() { Proxy = null };
var response2 = webClient.DownloadString("http://localhost:2006/1.aspx");

then response2 == "Hello world"

I need to achieve the same thing with a raw tcp connection

I am trying to achieve the same thing with a tcp connection and for some reason it does not work:

byte[] buf = new byte[1024];
string header = "GET http://localhost:2006/1.aspx HTTP/1.1\r\n" +
                "Host: localhost:2006\r\n" +
                "Connection: keep-alive\r\n" +
                "User-Agent: Mozilla/5.0\r\n" +
                "\r\n";

var client = new TcpClient("localhost", 2006);            

// send request
client.Client.Send(System.Text.Encoding.ASCII.GetBytes(header));

// get response
var i = client.Client.Receive(buf);
var response1 = System.Text.Encoding.UTF8.GetString(buf, 0, i);

here response1 != "Hello Wold". (note I use != meaning NOT equal)

In this example I get a bad request error.

---

I want to use a tcp connection for learning purposes. I dont understand why the second example does not work. My first reaction was maybe the headers are incorrect so what I did is I launched wireshark in order to see the headers send by my chrom browser. In fact the actual request sent by my browser when I goto http://localhost:2006/1.aspx is:

GET http://localhost:2006/1.aspx HTTP/1.1
Host: localhost:2006
Connection: keep-alive
Cache-Control: max-age=0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36
Accept-Encoding: gzip,deflate,sdch
Accept-Language: en-US,en;q=0.8

I have also tried using that request and when I do so I also get a Bad Request response! Why?

In other words I have replaced

string header = "GET http://localhost:2006/1.aspx HTTP/1.1\r\n" +
                "Host: localhost:2006\r\n" +
                "Connection: keep-alive\r\n" +
                "User-Agent: Mozilla/5.0\r\n" +
                "\r\n";

FOR

string header = "GET http://localhost:2006/1.aspx HTTP/1.1\r\n" +
        "Host: localhost:2006\r\n" +
        "Connection: keep-alive\r\n" +
        "Cache-Control: max-age=0\r\n" +
        "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" +
        "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36\r\n" +
        "Accept-Encoding: gzip,deflate,sdch\r\n" +
        "Accept-Language: en-US,en;q=0.8" +
        "\r\n\r\n";

and it still does not work.

Answer 1

Here is my version. Works fine

    private static async Task<string> HttpRequestAsync()
    {
        string result = string.Empty;

        using (var tcp = new TcpClient("www.bing.com", 80))
        using (var stream = tcp.GetStream())
        {
            tcp.SendTimeout = 500;
            tcp.ReceiveTimeout = 1000;
            // Send request headers
            var builder = new StringBuilder();
            builder.AppendLine("GET /?scope=images&nr=1 HTTP/1.1");
            builder.AppendLine("Host: www.bing.com");
            //builder.AppendLine("Content-Length: " + data.Length);   // only for POST request
            builder.AppendLine("Connection: close");
            builder.AppendLine();
            var header = Encoding.ASCII.GetBytes(builder.ToString());
            await stream.WriteAsync(header, 0, header.Length);

            // Send payload data if you are POST request
            //await stream.WriteAsync(data, 0, data.Length);

            // receive data
            using (var memory = new MemoryStream())
            {
                await stream.CopyToAsync(memory);
                memory.Position = 0;
                var data = memory.ToArray();

                var index = BinaryMatch(data, Encoding.ASCII.GetBytes("\r\n\r\n")) + 4;
                var headers = Encoding.ASCII.GetString(data, 0, index);
                memory.Position = index;

                if (headers.IndexOf("Content-Encoding: gzip") > 0)
                {
                    using (GZipStream decompressionStream = new GZipStream(memory, CompressionMode.Decompress))
                    using (var decompressedMemory = new MemoryStream())
                    {
                        decompressionStream.CopyTo(decompressedMemory);
                        decompressedMemory.Position = 0;
                        result = Encoding.UTF8.GetString(decompressedMemory.ToArray());
                    }
                }
                else
                {
                    result = Encoding.UTF8.GetString(data, index, data.Length - index);
                    //result = Encoding.GetEncoding("gbk").GetString(data, index, data.Length - index);
                }
            }

            //Debug.WriteLine(result);
            return result;
        }
    }

    private static int BinaryMatch(byte[] input, byte[] pattern)
    {
        int sLen = input.Length - pattern.Length + 1;
        for (int i = 0; i < sLen; ++i)
        {
            bool match = true;
            for (int j = 0; j < pattern.Length; ++j)
            {
                if (input[i + j] != pattern[j])
                {
                    match = false;
                    break;
                }
            }
            if (match)
            {
                return i;
            }
        }
        return -1;
    }