NodeJS. Dealing with � characters encoding

Question

I'm having difficulties dealing with character encoding. I'm trying to scrape the following url:

http://www.google.com/movies?near=Montreal&date=0

My code looks like this:

var http = require('http');
var url = require('url');
var Iconv  = require('iconv').Iconv;

var location = 'montreal';

var googleMovies = url.parse("http://www.google.com/movies?near=" + location);

var req = http.request(googleMovies, function(response) {
    var str = '';
    response.on('data', function(chunk) {
        str += chunk;
    });
    response.on('end', function() {

        var iconv = new Iconv('latin1', 'UTF-8');
        str = iconv.convert(str).toString();

        console.log(str);
    });
});
req.end()

I've first tried without:

    var iconv = new Iconv('latin1', 'UTF-8');
    str = iconv.convert(str).toString();

but that was causing the � characters.

I've tested the source listed above on this page:

http://nlp.fi.muni.cz/projects/chared/

and it seem to detect it as latin1, but things could be wrong.

Answer 1

The � characters come from the concatenation:

response.on('data', function(chunk) {
    str += chunk;
});

This converts each chunk to a String with the default encoding of utf8. Any sequences in the Buffers that aren't valid as UTF-8 will be lost and replaced by � at this point.

You'll want to leave the chunks as Buffers until after the convert(). They can be collected in an Array and combined with Buffer.concat().

var chunks = [];

response.on('data', function (chunk) {
    chunks.push(chunk);
});

response.on('end', function () {
    var iconv = new Iconv('latin1', 'UTF-8');
    var str = iconv.convert(Buffer.concat(chunks)).toString();
    console.log(str);
});

Answer 2

If you set your User-Agent to that of a desktop browser, the meta tag in the HTML and the Content-Type in the response headers will have the charset set to UTF-8 instead of latin1. Example:

var dest = url.parse('http://www.google.com/movies?near=montreal');
dest.headers = {
  'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36',
};

http.get(dest, function(response) {
  var str = '';

  response.on('data', function(chunk) {
    str += chunk;
  }).on('end', function() {
    console.log(str);
  }).setEncoding('utf8');
});