Javascript Web Workers File Upload

Question

I am trying to make a html uploader for very large files using HTML5 and Web Workers. Atm it uploads slowly but consumes alot of memory. I think it is transferring the whole file to the memory when it adds it to the form. Heres the code: jswebworker.js:

 /*importScripts('webworkerFormData.js');*/

(function() {
// Export variable to the global scope
(this == undefined ? self : this)['FormData'] = FormData;

var ___send$rw = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype['send'] = function(data) {
    if (data instanceof FormData) {
        if (!data.__endedMultipart) data.__append('--' + data.boundary + '--\r\n');
        data.__endedMultipart = true;
        this.setRequestHeader('Content-Type', 'multipart/form-data; boundary=' + data.boundary);
        data = new Uint8Array(data.data).buffer;
    }
    // Invoke original XHR.send
    return ___send$rw.call(this, data);
};

function FormData() {
    // Force a Constructor
    if (!(this instanceof FormData)) return new FormData();
    // Generate a random boundary - This must be unique with respect to the form's contents.
    this.boundary = '------RWWorkerFormDataBoundary' + Math.random().toString(36);
    var internal_data = this.data = [];
    /**
    * Internal method.
    * @param inp String | ArrayBuffer | Uint8Array  Input
    */
    this.__append = function(inp) {
        var i=0, len;
        if (typeof inp === 'string') {
            for (len=inp.length; i<len; i++)
                internal_data.push(inp.charCodeAt(i) & 0xff);
        } else if (inp && inp.byteLength) {/*If ArrayBuffer or typed array */
            if (!('byteOffset' in inp))   /* If ArrayBuffer, wrap in view */ 
                inp = new Uint8Array(inp);
            for (len=inp.byteLength; i<len; i++)
                internal_data.push(inp[i] & 0xff);
        }
    };
}
/**
* @param name     String                                  Key name
* @param value    String|Blob|File|Uint8Array|ArrayBuffer Value
* @param filename String                                  Optional File name (when value is not a string).
**/
FormData.prototype['append'] = function(name, value, filename) {
    if (this.__endedMultipart) {
        // Truncate the closing boundary
        this.data.length -= this.boundary.length + 6;
        this.__endedMultipart = false;
    }
    var valueType = Object.prototype.toString.call(value),
        part = '--' + this.boundary + '\r\n' + 
            'Content-Disposition: form-data; name="' + name + '"';

    if (/^\[object (?:Blob|File)(?:Constructor)?\]$/.test(valueType)) {

        return this.append(name,
                        new Uint8Array(new FileReaderSync().readAsArrayBuffer(value)),
                        filename || value.name);
    } else if (/^\[object (?:Uint8Array|ArrayBuffer)(?:Constructor)?\]$/.test(valueType)) {
        part += '; filename="'+ (filename || 'blob').replace(/"/g,'%22') +'"\r\n';
        part += 'Content-Type: application/octet-stream\r\n\r\n';
        this.__append(part);
        this.__append(value);
        part = '\r\n';
    } else {
        part += '\r\n\r\n' + value + '\r\n';
    }
    this.__append(part);
};
})();

movies = [];
var timeStarted = 0;
uploadingVar = false;
const BYTES_PER_CHUNK = 64 * 1024 * 1024; 

function toTitleCase(str)
{
return str.replace(/\w\S*/g, function(txt){return txt.charAt(0).toUpperCase() + txt.substr(1);});
}

function newUpload(blobOrFile, moviename, filename, i, fileType, sizeFile) {
var xhr = new XMLHttpRequest();
path = '/moviehtml/newmupload.php?        moviename='+escape(moviename)+'&filename='+escape(filename)+'&num='+escape(i);
xhr.open('POST', path, false);
self.postMessage(blobOrFile.size);
var fd = new FormData();
//xhr.setRequestHeader('Content-Type', fileType)
//blobOrFile2 = FileReaderSync.readAsArrayBuffer(blobOrFile);
fd.append("files1", blobOrFile);
//fd.append("moviename", moviename);
//fd.append("filename", filename);
//fd.append("num",i);

seconds = new Date() / 1000;
xhr.send(fd);
self.postMessage(xhr.responseText)
self.postMessage({'type':'partial','done':i*BYTES_PER_CHUNK, 'started':timeStarted, 'total':sizeFile});
var finish = Date()/1000;
if (finish >= (seconds+100)){
    return false;
}
return true;

}
function newFileUpload(file, movieName, fileType, filename, exten){
if (movieName == movieName.match(/[a-zA-Z0-9\. \:]+/)){
    timeStarted = new Date().getTime();

var blob = file;// this.files[0];
//var filename = blob.name;
var moviename = toTitleCase(movieName);
 // 1MB chunk sizes.
const SIZE = blob.size;
//alert(SIZE + ' '+document.getElementById('fileToUpload').files[0].size)
var start = 0;
var end = BYTES_PER_CHUNK;
//alert(SIZE/BYTES_PER_CHUNK)
var i = 1;
while(start < SIZE) {
    wow = newUpload(blob.slice(start, end), moviename, filename, i, fileType, SIZE);

    start = end;
    end = start + BYTES_PER_CHUNK;
    i++;
}
var xhr = new XMLHttpRequest();
var fd2 = new FormData();
typeFile = filename.split('.').pop()
fd2.append("type", blob.type);
fd2.append("exten", typeFile);
fd2.append("moviename", moviename);
xhr.open('POST', '/moviehtml/finishedupload.php', false);
xhr.send(fd2);
}
}
function process(){
uploadingVar = true;
while(movies.length > 0) {

    upMovie = movies.pop();
    var file = upMovie[0];
    var movieName = upMovie[1];
    var fileType = upMovie[2];
    var filename = upMovie[3];
    var exten = upMovie[4];
    self.postMessage({'type':'start','size':file.size, 'name':movieName});

    newFileUpload(file, movieName, fileType, filename, exten);
    self.postMessage({'type':'finish','name':movieName})
    self.postMessage(movieName + " Uploaded Succesfully");
 }
 uploadingVar = false;
}

self.addEventListener('message', function(e) {
movies.push(e.data);
if (!uploadingVar){
process();

}
}, false);

My function that calls it:

var worker = new Worker('jswebworker.js');
function testUpload(){
//newFileUpload();
var file = document.getElementById('fileToUpload').files[0];
worker.postMessage([file,toTitleCase(document.getElementById('movieName').value),     file.type, file.name, file.name.split('.').pop()]);

}

This is the web page for a media server for my flat. I am hoping that there is a way to create a blob without loading all of the original into memory. Thanks for any help, Nick

Answer 1

I think that this library (here is the github page for resumable.js as well) may already be doing what you are trying too accomplish. Since I don't know how big the files are that you are tying to upload I can't provide any insight as far as benchmarks go. I hope this helps you out.