How can I make a POST request in the same manner as Apache HttpClient lib, using HttpURLConnection

Question

I'm trying to make a POST request to a website. As the response to the POST request, I expect some JSON data.

Using Apache's HttpClient library, I am able to do this without any problems. The response data is JSON so I just parse it.

package com.mydomain.myapp;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class MyApp {

    private static String extract(String patternString, String target) {

        Pattern pattern = Pattern.compile(patternString);
        Matcher matcher = pattern.matcher(target);
        matcher.find();
        return matcher.group(1);
    }

    private String getResponse(InputStream stream) throws Exception {

        BufferedReader in = new BufferedReader(new InputStreamReader(stream));
        String inputLine;
        StringBuffer responseStringBuffer = new StringBuffer();
        while ((inputLine = in.readLine()) != null) {
            responseStringBuffer.append(inputLine);
        }
        in.close();
        return responseStringBuffer.toString();
    }

    private final static String BASE_URL = "https://www.volkswagen-car-net.com";
    private final static String BASE_GUEST_URL = "/portal/en_GB/web/guest/home";

    private void run() throws Exception {

        CloseableHttpClient client = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(BASE_URL + BASE_GUEST_URL);
        CloseableHttpResponse getResponse = client.execute(httpGet);
        HttpEntity responseEntity = getResponse.getEntity();
        String data = getResponse(responseEntity.getContent());
        EntityUtils.consume(responseEntity);

        String csrf = extract("<meta name=\"_csrf\" content=\"(.*)\"/>", data);
        System.out.println(csrf);

        HttpPost post = new HttpPost(BASE_URL + "/portal/web/guest/home/-/csrftokenhandling/get-login-url");
        post.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        post.setHeader("User-Agent'", "Mozilla/5.0 (Linux; Android 6.0.1; D5803 Build/23.5.A.1.291; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.111 Mobile Safari/537.36");
        post.setHeader("Referer", BASE_URL + "/portal");
        post.setHeader("X-CSRF-Token", csrf);

        CloseableHttpResponse postResponse = client.execute(post);
        HttpEntity postResponseEntity = postResponse.getEntity();
        String postData = getResponse(postResponseEntity.getContent());
        System.out.println(postData);
        EntityUtils.consume(postResponseEntity);
        postResponse.close();
    }

    public static void main(String[] args) throws Exception {

        MyApp myApp = new MyApp();
        myApp.run();
    }
}

But I can't use the HttpClient library in my project. I need to be able to do the same thing with "just" HttpURLConnection.

But there is some magic going on with the HttpClient library that I cannot fathom. Because the response to my POST request using HttpURLConnection is just a redirect to a different webpage alltogheter.

Can someone point me in the right direction here?

Here's my current HttpURLConnection attempt:

package com.mydomain.myapp;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class MyApp {

    private static String extract(String patternString, String target) {

        Pattern pattern = Pattern.compile(patternString);
        Matcher matcher = pattern.matcher(target);
        matcher.find();
        return matcher.group(1);
    }

    private final static String BASE_URL = "https://www.volkswagen-car-net.com";
    private final static String BASE_GUEST_URL = "/portal/en_GB/web/guest/home";

    private String getResponse(InputStream stream) throws Exception {

        BufferedReader in = new BufferedReader(new InputStreamReader(stream));
        String inputLine;
        StringBuffer responseStringBuffer = new StringBuffer();
        while ((inputLine = in.readLine()) != null) {
            responseStringBuffer.append(inputLine);
        }
        in.close();
        return responseStringBuffer.toString();
    }

    private String getResponse(HttpURLConnection connection) throws Exception {
        return getResponse(connection.getInputStream());
    }

    private void run() throws Exception {

        HttpURLConnection getConnection1;
        URL url = new URL(BASE_URL + BASE_GUEST_URL);
        getConnection1 = (HttpURLConnection) url.openConnection();
        getConnection1.setRequestMethod("GET");
        if (getConnection1.getResponseCode() != HttpURLConnection.HTTP_OK) {
            throw new Exception("Request failed");
        }

        String response = getResponse(getConnection1);
        getConnection1.disconnect();

        String csrf = extract("<meta name=\"_csrf\" content=\"(.*)\"/>", response);
        System.out.println(csrf);

        HttpURLConnection postRequest;
        URL url2 = new URL(BASE_URL + "/portal/web/guest/home/-/csrftokenhandling/get-login-url");
        postRequest = (HttpURLConnection) url2.openConnection();
        postRequest.setDoOutput(true);
        postRequest.setRequestMethod("POST");
        postRequest.setInstanceFollowRedirects(false);

        postRequest.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        postRequest.setRequestProperty("User-Agent'", "Mozilla/5.0 (Linux; Android 6.0.1; D5803 Build/23.5.A.1.291; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.111 Mobile Safari/537.36");
        postRequest.setRequestProperty("Referer", BASE_URL + "/portal");
        postRequest.setRequestProperty("X-CSRF-Token", csrf);

        postRequest.disconnect();
    }

    public static void main(String[] args) throws Exception {

        MyApp myApp = new MyApp();
        myApp.run();
    }
}

Answer 1

Courtesy of a great programmer resource, e.g. MKYong (you know you've run into his site before ;-)) and I'll go over the gist of it in case the link ever goes down.

Gist:

The HttpURLConnection‘s follow redirect is just an indicator, in fact it won’t help you to do the “real” http redirection, you still need to handle it manually.
If a server is redirected from the original URL to another URL, the response code should be 301: Moved Permanently or 302: Temporary Redirect. And you can get the new redirected url by reading the “Location” header of the HTTP response header.

For example, access to the normal HTTP twitter website – http://www.twitter.com , it will auto redirect to the HTTPS twitter website – https://www.twitter.com.

Sample code

package com.mkyong.http;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class HttpRedirectExample {

  public static void main(String[] args) {

    try {

    String url = "http://www.twitter.com";

    URL obj = new URL(url);
    HttpURLConnection conn = (HttpURLConnection) obj.openConnection();
    conn.setReadTimeout(5000);
    conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
    conn.addRequestProperty("User-Agent", "Mozilla");
    conn.addRequestProperty("Referer", "google.com");

    System.out.println("Request URL ... " + url);

    boolean redirect = false;

    // normally, 3xx is redirect
    int status = conn.getResponseCode();
    if (status != HttpURLConnection.HTTP_OK) {
        if (status == HttpURLConnection.HTTP_MOVED_TEMP
            || status == HttpURLConnection.HTTP_MOVED_PERM
                || status == HttpURLConnection.HTTP_SEE_OTHER)
        redirect = true;
    }

    System.out.println("Response Code ... " + status);

    if (redirect) {

        // get redirect url from "location" header field
        String newUrl = conn.getHeaderField("Location");

        // get the cookie if need, for login
        String cookies = conn.getHeaderField("Set-Cookie");

        // open the new connnection again
        conn = (HttpURLConnection) new URL(newUrl).openConnection();
        conn.setRequestProperty("Cookie", cookies);
        conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
        conn.addRequestProperty("User-Agent", "Mozilla");
        conn.addRequestProperty("Referer", "google.com");

        System.out.println("Redirect to URL : " + newUrl);

    }

    BufferedReader in = new BufferedReader(
                              new InputStreamReader(conn.getInputStream()));
    String inputLine;
    StringBuffer html = new StringBuffer();

    while ((inputLine = in.readLine()) != null) {
        html.append(inputLine);
    }
    in.close();

    System.out.println("URL Content... \n" + html.toString());
    System.out.println("Done");

    } catch (Exception e) {
    e.printStackTrace();
    }

  }

}