NodeJS: Uploading a remote file to S3 with request and knox

Question

I'm trying to devise a way to upload a file from a url to s3 using request and knox. Currently, my code looks like this:

request(item.productImage, function(err, res, body) {
  if (!err && res.statusCode == 200) {
    fs.writeFile('/tmp/' + filename, body, 'base64', function(err, data){
      if (err) { 
        return console.log(err);
      }

      client.putFile('/tmp/' + filename, '/item/' + item._id + '/' + filename, function(err, res) {
        if (err) {
          return console.log(err);
        }
      }); 
    });
  }
});

This doesn't work as it downloads about 652 bytes of a 4kb file before it stops. Strangely, if I don't provide a callback to fs.writeFile() it downloads the entire 4kb locally.

What's the best way of accomplishing this?

Answer 1

There are a number of questions about this here on Stackoverflow, but I can't seem to find one that answers your question. The solution below should work, however, I'm having trouble getting knox to work at all on my machine right now. I hope you will be more lucky!

UPDATE: I seem to have had some problems with s3 here, the code below works -- I did change one thing, you need to specify encoding as null to request, so you get a Buffer back. Otherwise, binary data won't work so well.

request(item.productImage, {encoding: null}, function(err, res, body) {
  if(!err && res.statusCode == 200) {
    var req = client.put('/item/' + item._id + '/' + filename, {
      'Content-Type': res.headers['content-type'],
      'Content-Length': res.headers['content-length']
    });

    req.on('response', function(res) {
      console.log('response from s3, status:', res.statusCode, 'url:', req.url);
    });

    req.on('error', function(err) {
      console.error('Error uploading to s3:', err);
    });

    req.end(body);
  }
});

Note: With this solution, you avoid having to buffer the files to disk - that's why I chose to use the lower level put method of the knox client.