request node module not giving html

Question

I am using request nodejs module to get html for a website as below:

var request = require('request');

request("http://www.thenewschool.org/", function (error, response, body) {
    if (!error && response.statusCode == 200) {
        console.log("body>>>>>>>>>>");
    } else {
        console.log("error>>>>>>>>>"+error);
        console.log("response statusCode>>>>>>>>>"+response.statusCode);
        console.log("response body>>>>>>>>>"+response.body);
    }
})

and this gives me this output

error>>>>>>>>>null

response statusCode>>>>>>>>>403

response body>>>>>>>>>Sorry, this request has been blocked due to an invalid user agent.

This is passing for most of the cases, but fails in this case, can someone help me to solve this.

Answer 1

You just have to pass the user-agent in the headers (because the URL requires it), like:

var options = {
  headers: {'user-agent': 'node.js'}
}

request("http://www.thenewschool.org/", options, function (error, response, body) {
  if (!error && response.statusCode == 200) {
    console.log("body>>>>>>>>>>" + body);
  } else {
    console.log("error>>>>>>>>>"+error);
    console.log("response statusCode>>>>>>>>>"+response.statusCode);
    console.log("response body>>>>>>>>>"+response.body);
  }
})