streams with percentage complete

Question

I need to stream a file in base64 to an http endpoint using something like request or superagent. What is the best way to figure out what percentage of the file has been uploaded?

I assume I can create the read stream using something like:

fs.createReadStream('/tmp/cats.jpg', {encoding: 'base64'})

Any examples using one out of above libraries would be greatly appreciated.

Answer 1

I think you can use progress-stream.

Here is an example from the package:

var progress = require('progress-stream');
var fs = require('fs');
 
var stat = fs.statSync(filename);
var str = progress({
    length: stat.size,
    time: 100 /* ms */
});
 
str.on('progress', function(progress) {
    console.log(progress);
 
    /*
    {
        percentage: 9.05,
        transferred: 949624,
        length: 10485760,
        remaining: 9536136,
        eta: 42,
        runtime: 3,
        delta: 295396,
        speed: 949624
    }
    */
});
 
fs.createReadStream(filename)
    .pipe(str)
    .pipe(fs.createWriteStream(output));

Answer 2

I was looking for an answer to a similar issue and thanks to Alberto Zaccagni's answer, I was able to get some code working.

So for the people who don't want to piece the puzzle themselves, here is the code (edited for Stackoverflow):

var zipfile = "my_large_archive.zip";

// Get the size of the file
fs.stat(zipfile, function (err, stats) {
    var zipSize         = stats.size;
    var uploadedSize    = 0; // Incremented by on('data') to keep track of the amount of data we've uploaded

    // Create a new read stream so we can plug events on it, and get the upload progress
    var zipReadStream   = fs.createReadStream(zipfile);

    zipReadStream.on('data', function(buffer) {
        var segmentLength   = buffer.length;

        // Increment the uploaded data counter
        uploadedSize        += segmentLength;

        // Display the upload percentage
        console.log("Progress:\t",((uploadedSize/zipSize*100).toFixed(2)+"%"));
    });

    // Some other events you might want for your code
    zipReadStream.on('end', function() {
        console.log("Event: end");
    });
    zipReadStream.on('close', function() {
        console.log("Event: close");
    });


    var formData    = require('form-data');
    var form        = new formData();

    form.append('apikey', 'f4sd5f4sdf6ds456');  // Just some post parameters I need to send to the upload endpoint
    form.append('file', zipReadStream);         // The zip file, passed as a fs.createReadStream instance

    // Submit the form and the file
    form.submit('http://www.someserver.com/upload', function(err, res) {
        if (err) {
            console.log("Oups! We encountered an error :(\n\n", err);
            return false;
        }
        console.log("Your file has been uploaded.");
        res.resume();   // Fix is you use that code for a CLI, so that the execution will stop and let users enter new commands
    });
});