Get upload progress with Meteor

Question

I was about to use XHR to send a file to the server when I realized that I don't have an end-point due to lack of routing.

Then I read this article and discovered I could potentially do file uploads by taking advantage of Meteor.methods. Now my upload looks something like this:

$(function() {
    $(document.body).html(Meteor.render(Template.files));
    $(document).on('drop', function(dropEvent) {
        _.each(dropEvent.originalEvent.dataTransfer.files, function(file) {
            var reader = new FileReader();
            reader.onload = function(fileLoadEvent) {
                Meteor.call('uploadFile', file, reader.result);
            };
            reader.readAsBinaryString(file);
        });
        dropEvent.preventDefault();
    });

    $(document).bind('dragover dragenter', function(e) {
        e.preventDefault();
    });

});

And in server/main.js I have this:

var require = __meteor_bootstrap__.require; // should I even be doing this? looks like an internal method
var fs = require('fs');
var path = require('path');

Meteor.methods({
    uploadFile: function(fileInfo, fileData) {
        var fn = path.join('.uploads',fileInfo.name);
        fs.writeFile(fn, fileData, 'binary', function(err) {
            if(err) {
                throw new Meteor.Error(500, 'Failed to save file.', err);
            } else {
                console.log('File saved to '+fn);
            }
        });
    }
});

Which just writes it to disk. This seems to work, but I don't know what technology Meteor is using to pass the data to that method on the server, and I don't know how to get progress info back.

Normally I'd attach an event listener to the xhr object,

xhr.upload.addEventListener("progress", uploadProgress, false);

but I don't think I have access to one with .methods. Is there another way to do this?

Answer 1

I've been working on this as well, and I have some code that works.

To clarify first:

This seems to work, but I don't know what technology Meteor is using to pass the data to that method on the server, and I don't know how to get progress info back.

Meteor has a WebSockets connection with the server and it uses this as a transport. So every time you call Meteor.call or Meteor.apply it will EJSON encode your parameters (if any), call the function server-side, and return you the response transparently.

The trick here is to then use HTML5's FileReader Api to read the file in chunks (important because large files will crash your browser otherwise) and perform a Meteor.call() for every chunk.

Your code is doing two things that didn't work well for me:

1. It's reading the file as a Binary String: I found that using Base64 is a lot more reliable. You say your code uploads, but have you checked file checksums to make sure they're the same file?
2. It's reading the file all at once. Large files (100s of MB) will cause performance issues, and in my experience even crash your browser.

Ok, to your question now. How to tell how much of the file you've uploaded? You can make your chunks small enough so that you can use chunks_sent / total_chunks as a metric for that. Alternatively you could call Session.set( 'progress', current_size / total_size ) from the Meteor server-side call and bind it to an element so that it updates.

This is a jQuery plugin that I've been working on to wrap this functionality. It's not complete, but it uploads files and it may be helpful to you. It currently only gets files via drag drop.. there's no "browse" button.

Disclaimer: I am quite new to Meteor & node, so some things may not be done the "recommended" way, but I'll improve it over time and eventually give it a home in Github.

;(function($) {

    $.uploadWidget = function(el, options) {

        var defaults = {
            propertyName: 'value',
            maximumFileSize: 1073741824, //1GB

            messageTarget: null
        };

        var plugin = this;

        plugin.settings = {}


        var init = function() {
            plugin.settings = $.extend({}, defaults, options);
            plugin.el = el;

            if( !$(el).attr('id') || !$(el).attr('id').length ){
                $(el).attr('id', 'uploadWidget_' + Math.round(Math.random()*1000000));
            }

            if( plugin.settings.messageTarget == null ){
                plugin.settings.messageTarget = plugin.el;
            }

            initializeDropArea();
        };


        // Returns a human-friendly string representation of bytes
        var getBytesAsPrettyString = function( bytes ){

            var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
            if (bytes == 0) return 'n/a';
            var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)));
            return Math.round(bytes / Math.pow(1024, i), 2) + ' ' + sizes[i];

        };


        // Throws an exception if the file (from a drop event) is unacceptable
        var assertFileIsAcceptable = function( file ){

            if( file.size > plugin.settings.maximumFileSize ){
                throw 'Files can\'t be larger than ' + getBytesAsPrettyString(plugin.settings.maximumFileSize);
            }

            //if( !file.name.match(/^.+?\.pdf$/i) ){
            //    throw 'Only pdf files can be uploaded.';
            // }

        };


        var setMainMessage = function( message ){

            $(plugin.settings.messageTarget).text( message );

        };


        plugin.getUploader = function(){

            return plugin.uploader;

        };


        var initializeDropArea = function(){

            var $el = $(plugin.el);

            $.event.props.push("dataTransfer");

            $el.bind( 'dragenter dragover dragexit', function(){
                event.stopPropagation();
                event.preventDefault();
            });

            $el.bind( 'drop', function( event ){

                var slices;
                var total_slices;

                var processChunkUpload = function( blob, index, start, end ){

                    var chunk;

                    if (blob.webkitSlice) {
                        chunk = blob.webkitSlice(start, end);
                    } else if (blob.mozSlice) {
                        chunk = blob.mozSlice(start, end);
                    } else {
                        chunk = blob.slice(start,end);
                    }

                    var reader = new FileReader();

                    reader.onload = function(event){

                        var base64_chunk = event.target.result.split(',')[1];

                        slices--;

                        $el.text( slices + ' out of ' + total_slices + ' left' );

                        Meteor.apply(
                            'saveUploadFileChunk',
                            [file_name, base64_chunk, slices+1],
                            { wait: true }
                        );
                    };

                    reader.readAsDataURL(chunk);
                };


                event.stopPropagation();
                event.preventDefault();
                event.dataTransfer.dropEffect = 'copy';

                if( !event.dataTransfer.files.length ){
                    return;
                }

                const BYTES_PER_CHUNK = 1024 * 1024; // 1MB chunk sizes.


                var blob = event.dataTransfer.files[0];
                var file_name = blob.name;

                var start = 0;
                var end;
                var index = 0;

                // calculate the number of slices we will need
                slices = Math.ceil(blob.size / BYTES_PER_CHUNK);
                total_slices = slices;

                while(start < blob.size) {
                    end = start + BYTES_PER_CHUNK;
                    if(end > blob.size) {
                        end = blob.size;
                    }

                    processChunkUpload( blob, index, start, end );

                    start = end;
                    index++;
                }


            });

        };

        init();

    }

})(jQuery);

And this is my meteor published method.

Meteor.methods({

        // this is TOTALLY insecure. For demo purposes only.
        // please note that it will append to an existing file if you upload a file by the same name..
        saveUploadFileChunk: function ( file_name, chunk, chunk_num ) {

                var require = __meteor_bootstrap__.require;
                var fs = require('fs');
                var crypto = require('crypto')

                var shasum = crypto.createHash('sha256');
                shasum.update( file_name );

                var write_file_name = shasum.digest('hex');

                var target_file = '../tmp/' + write_file_name;

                fs.appendFile(
                        target_file,
                        new Buffer(chunk, 'base64'),
                        {
                                encoding: 'base64',
                                mode: 438,
                                flag: 'a'
                        }
                        ,function( err ){

                                if( err ){

                                        console.log('error ' + err);
                                }

                                console.log( 'wrote ' + chunk_num );

                        }
                );

                return write_file_name;

        }
});

HTH