Node.js request download file sometimes empty or no existing

Question

I'm writing a downloader with node.js and the request module. Using the stream syntax I'm doing

var r = request(url).pipe(fs.createWriteStream(targetPath));
r.on('error', function(err) { console.log(err); });
r.on('finish', cb);

to download the file, save it and call the callback. However, in almost 50% of the cases the file is either empty or not created at all. No error event is emitted. It seems like the finish event is triggered even though the file wasn't (completely) written yet.

Context: The whole thing is wrapped into async.each calls.

Any clues? Thanks!

Answer 1

You need to close the file before accessing it:

var file = fs.createWriteStream(targetPath);
var r = request(url).pipe(file);
r.on('error', function(err) { console.log(err); });
r.on('finish', function() { file.close(cb) });

Incidentally, if the url replies with any http error (such as a 404 not found), that won't trigger the 'error' event, so you should probably check that separately:

function handleFailure(err) { console.log(err); };

var file = fs.createWriteStream(targetPath);
request(url, function(error, response) {
    if (response.statusCode != 200) {
        console.log("oops, got a " + response.statusCode);
        return
    }
    // close is async, and you have to wait until close completes otherwise
    // you'll (very rarely) not be able to access the file in the callback.
    file.on('finish', function() { file.close(cb) });

    response.pipe(file).on('error', handleFailure)
    file.on('error', handleFailure)
}).on('error', handleFailure);