Java Proxy - can't exchange data from HTTP GET/POST request properly

Question

Starting from this post I tried to implement a little proxy server that handles GET and POST requests as well (just replace the Handler class by the one below):

public static class Handler extends Thread {

    public static final Pattern CONNECT_PATTERN
            = Pattern.compile("CONNECT (.+):(.+) HTTP/(1\\.[01])", Pattern.CASE_INSENSITIVE);
    public static final Pattern GET_POST_PATTERN
            = Pattern.compile("(GET|POST) (?:http)://([^/:]*)(?::([^/]*))?(/.*) HTTP/(1\\.[01])", Pattern.CASE_INSENSITIVE);

    private final Socket clientSocket;
    private boolean previousWasR = false;

    public Handler(Socket clientSocket) {
        this.clientSocket = clientSocket;
    }

    @Override
    public void run() {
        try {
            String request = readLine(clientSocket, Integer.MAX_VALUE);

            Matcher connectMatcher = CONNECT_PATTERN.matcher(request);
            Matcher getNpostMatcher = GET_POST_PATTERN.matcher(request);

            System.out.println("Request: " +request);
            if (connectMatcher.matches()) {
                // ...

            } else if (getNpostMatcher.matches()) {
                String method = getNpostMatcher.group(1);
                String hostString = getNpostMatcher.group(2);
                String portString = getNpostMatcher.group(3);
                String lengthString = null;
                String line;
                ArrayList<String> buffer = new ArrayList<String>();
                Integer port = portString == null || "".equals(portString) ? 80 : Integer.parseInt(portString);
                Integer length = null;

                buffer.add(request);
                while ((line = readLine(clientSocket, Integer.MAX_VALUE)) != null) {
                    buffer.add(line);

                    if ("".equals(line)) break;

                    if (lengthString == null && line.startsWith("Content-Length: ")) {
                        lengthString = line.substring(16);
                        length = Integer.parseInt(lengthString);
                    }
                }

                try {
                    final Socket forwardSocket;
                    try {
                        forwardSocket = new Socket(hostString, port);
                        System.out.println("  " + forwardSocket);

                    } catch (IOException | NumberFormatException e) {
                        OutputStreamWriter outputStreamWriter
                                = new OutputStreamWriter(clientSocket.getOutputStream(), "ISO-8859-1");

                        e.printStackTrace();
                        outputStreamWriter.write("HTTP/" + connectMatcher.group(3) + " 502 Bad Gateway\r\n");
                        outputStreamWriter.write("Proxy-agent: Simple/0.1\r\n");
                        outputStreamWriter.write("\r\n");
                        outputStreamWriter.flush();
                        return;
                    }

                    PrintWriter printWriter = new PrintWriter(forwardSocket.getOutputStream());

                    for (String bufferedLine : buffer) {
                        printWriter.println(bufferedLine);
                    }
                    printWriter.flush();

                    if ("POST".equals(method) && length > 0) {
                        System.out.println ("Posting data ...");
                        if (previousWasR) { // skip \n if existing
                            int read = clientSocket.getInputStream().read();
                            if (read != '\n') {
                                forwardSocket.getOutputStream().write(read);
                            }
                            forwardData(threadId, clientSocket, forwardSocket, length, true); // only forward "Content-length" bytes
                        } else {
                            forwardData(threadId, clientSocket, forwardSocket, length, true); // only forward "Content-length" bytes
                        }
                    } 

                    System.out.println ("Forwarding response ...");
                    forwardData(threadId, forwardSocket, clientSocket, null, false);

                    if (forwardSocket != null) {
                        forwardSocket.close();
                    }

                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                clientSocket.close();

            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    private static void forwardData(int threadId, Socket inputSocket, Socket outputSocket, Integer length, boolean isPost) {
        try {
            InputStream inputStream = inputSocket.getInputStream();
            try {
                OutputStream outputStream = outputSocket.getOutputStream();
                try {
                    byte[] buffer = new byte[4096];
                    int read;
                    if (length == null || length > 0) {
                        do {
                            if ((read = inputStream.read(buffer)) > 0) {
                                outputStream.write(buffer, 0, read);
                                if (inputStream.available() < 1) {
                                    outputStream.flush();
                                }
                                if (length != null) {
                                    length = length - read;
                                }
                            }
                        } while (read >= 0 && (length == null || length > 0));
                    }
                } finally {
                    if (!outputSocket.isOutputShutdown()) {
                        if (!isPost) {
                            outputSocket.shutdownOutput();
                        }
                    }
                }
            } finally {
                if (!inputSocket.isInputShutdown()) {
                    inputSocket.shutdownInput();
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private String readLine(Socket socket, Integer noOfBytes) throws IOException {
        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
        int next;
        readerLoop:
        while (noOfBytes-- > 0 && (next = socket.getInputStream().read()) != -1) {
            if (previousWasR && next == '\n') {
                previousWasR = false;
                continue;
            }
            previousWasR = false;
            switch (next) {
                case '\r':
                    previousWasR = true;
                    break readerLoop;
                case '\n':
                    break readerLoop;
                default:
                    byteArrayOutputStream.write(next);
                    break;
            }
        }
        return byteArrayOutputStream.toString("ISO-8859-1");
    }
}

After having some issues with POST requests, I figured out that it is important to shut down the streams in the right way. So finally the code above is working quite well when using the Internet Explorer.

Using other browsers, however, it looks like the streams/sockets are not properly closed, because sometimes loading indicators are running for quite a while, though the content seems already to be loaded. Sometimes, sites are not completely loaded and threads seem to hang at ...

if ((read = inputStream.read(buffer)) > 0) {

in forwardData(...). I don't know how I could find out, whether the stream might provide some data or not - or how to avoid this blocking call of read at all.

Does anybody know what I'm doing wrong and how I could forward the data properly, so that all browsers load contents correctly without unnecessary delays?

Answer 1

Look, the problem is that you are using forwardData() to copy server responses to the client, because it doesn't involve the Content-Length: field.

HTTP/1.1{HTTP/1.1 200 OK
Cache-Control: no-cache, must-revalidate
Pragma: no-cache
Content-Type: text/html; Charset=utf-8
Content-Encoding: gzip
Expires: Fri, 01 Jan 1990 00:00:00 GMT
Vary: Accept-Encoding
Server: Microsoft-IIS/8.0
Access-Control-Allow-Origin: *
X-RADID: P301511016-T104210110-C24000000000248666
P3P: CP="BUS CUR CONo FIN IVDo ONL OUR PHY SAMo TELo"
Date: Tue, 15 Dec 2015 09:33:41 GMT
Content-Length: 1656

<1656 Bytes>

In the example response above, the proxy has to shutdown the streams after reading 1656 bytes from the server. If it does not, read will block.

So what you need is some code that searches the response for the Content-Length: field, for instance:

private static void forwardData(int threadId, Socket inputSocket, Socket outputSocket, Integer length, boolean isPost) {
    int cLength = -1;
    int count = 0;
    try {
        InputStream inputStream = inputSocket.getInputStream();
        try {
            OutputStream outputStream = outputSocket.getOutputStream();
            try {
                byte[] buffer = new byte[4096];
                int read;
                if (length == null || length > 0) {
                    do {
                        if ((read = inputStream.read(buffer)) > 0) { // search for "Content-Length: "
                            if (cLength == -1) {
                                String response = new String(buffer, "UTF-8");
                                int pos = response.indexOf("Content-Length:");
                                if (pos > 0) {
                                    String lString = response.substring(pos + 16, pos + 24).replaceAll("([0-9]*).*\\n?\\r?.*", "$1");
                                    cLength = Integer.parseInt(lString);
                                }
                            }

                            if (cLength != -1) { // if length is given, count bytes from empty line on
                                if (count > 0) { // already started  - so just add
                                    count = count + read;
                                } else { // check if empty line exists,  "\r\n\r\n" or "\r\r"
                                    for (int n = 0; n < read; n++) {
                                        if (buffer[n] == 13 && buffer[n + 1] == 13) {
                                            count = read - (n + 2); // if so, set count to bytes read after the empty line
                                        }
                                        if (buffer[n] == 13 && buffer[n + 1] == 10 && buffer[n + 2] == 13 && buffer[n + 3] == 10) {
                                            count = read - (n + 4); // same as above
                                        }
                                    }
                                }
                            }

                            outputStream.write(buffer, 0, read);
                            if (inputStream.available() < 1) {
                                outputStream.flush();
                            }
                            if (length != null) {
                                length = length - read;
                            }
                        }
                    } while (read >= 0 && (length == null || length > 0) && (cLength == -1 || count < cLength));
                }

            } finally {
                if (!outputSocket.isOutputShutdown()) {
                    if (!isPost) {
                        outputSocket.shutdownOutput();
                    }
                }
            }
        } finally {
            if (!inputSocket.isInputShutdown()) {
                inputSocket.shutdownInput();
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
}

Though the code above is currently not 100% working, it should give you an idea on how to proceed. If content length is null and an empty line is received you've probably to shutdown the streams as well.