CasperJS File Download Times Out After 30 Seconds

Question

I'm using CasperJS to download a 15 MB file. In the browser, the download takes about 3 minutes to complete. With Casper, the .download function for the same url returns after exactly 30 seconds, and the file written to disk is 0 bytes. I've tried setting longer timeouts like this:

var casper = require("casper").create({
    pageSettings: {
        webSecurityEnabled: false
    },
    waitTimeout: 500000,
    stepTimeout: 500000
});

But they have no effect. Here's my download function:

casper.on('resource.received', function (resource) {
    var url, file;
    if ((resource.url.indexOf("myDownloadUniqueString=") !== -1) ) {
        this.echo(resource.url);  // the echo'ed url can be downloaded in a web browser
        url = resource.url;
        file = "downloaded_file.wav";  // this will be 0 bytes
        try {
            var fs = require("fs"); // phantom js file system (not node)
            casper.download(resource.url, file);
        } catch (e) {
            this.echo(e);  // no error is thrown
        }
    }
});

Any ideas? Perhaps an issue with the PhantomJS fs methods, but that documentation is incomplete...

Answer 1

I solved this problem (for an Excel .xls binary download taking >30s, 6Mb approx) by running an async XMLHTTPrequest (xhr) manually within an evaluate function, then writing the result to a global (window) variable, and waiting for this global to be set.

On the xhr object, you can set a custom timeout, 15 * 60 * 1000 = 15 mins in the example below.

Care needs to be taken with encoding binary downloads as ascii / base64, and then decoding them to write a binary file. This could be adapted / simplified for text downloads.

var fs = require('fs');
var casper = require('casper').create({
    //options here
});

var xhr = this.evaluate(function(url){

    var xhr = new XMLHttpRequest();
    xhr.timeout = 15 * 60 * 1000;
    xhr.overrideMimeType("text/plain; charset=x-user-defined");
    xhr.open("GET", url);  // synchronous request banned, so use waitfor to wait on a global variable

    xhr.onreadystatechange = function () {
        if (xhr.readyState == 4) {
            //if (xhr.status == 200) { 
            window.xhrstatus = xhr.status;    //write to a global (window) variable
            window.xhrResponseText = __utils__.encode(xhr.responseText);   //base64 encode using casper functions (btoa fails)
            //}
        }
    };                        

    xhr.send(null);
    return true;
},'http://example.com/download.xls');


casper.waitFor(function() { 
    return this.getGlobal('xhrstatus') != undefined; 
}, function() { 
    this.echo('XHR status: ' + this.getGlobal('xhrstatus')); 
    this.echo('Saving report...'); 
    //http://phantomjs.org/api/fs/method/is-writable.html to check if file writable first
    //decode using casper clientutil function and then write binary file
    fs.write('saveFileName.xls', decode(this.getGlobal('xhrResponseText')), 'wb');
},null,15*60*1000);

The encode / decode functions from casper.js clientutils library look like this. These seem to work where Javascript's atob() and btoa() don't.

/* 
 * encode / decode function from casper.js clientutils 
 * https://github.com/casperjs/casperjs/blob/master/modules/clientutils.js
 * Included here for reference - you could just reference the file in your code
*/
var BASE64_ENCODE_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
var BASE64_DECODE_CHARS = [
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63,
    52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1,
    -1,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
    15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1,
    -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
    41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1
];

/**
 * Decodes a base64 encoded string. Succeeds where window.atob() fails.
 *
 * @param  String  str  The base64 encoded contents
 * @return string
 */
var decode = function decode(str) {
    /*eslint max-statements:0, complexity:0 */
    var c1, c2, c3, c4, i = 0, len = str.length, out = "";
    while (i < len) {
        do {
            c1 = BASE64_DECODE_CHARS[str.charCodeAt(i++) & 0xff];
        } while (i < len && c1 === -1);
        if (c1 === -1) {
            break;
        }
        do {
            c2 = BASE64_DECODE_CHARS[str.charCodeAt(i++) & 0xff];
        } while (i < len && c2 === -1);
        if (c2 === -1) {
            break;
        }
        out += String.fromCharCode(c1 << 2 | (c2 & 0x30) >> 4);
        do {
            c3 = str.charCodeAt(i++) & 0xff;
            if (c3 === 61) {
                return out;
            }
            c3 = BASE64_DECODE_CHARS[c3];
        } while (i < len && c3 === -1);
        if (c3 === -1) {
            break;
        }
        out += String.fromCharCode((c2 & 0XF) << 4 | (c3 & 0x3C) >> 2);
        do {
            c4 = str.charCodeAt(i++) & 0xff;
            if (c4 === 61) {
                return out;
            }
            c4 = BASE64_DECODE_CHARS[c4];
        } while (i < len && c4 === -1);
        if (c4 === -1) {
            break;
        }
        out += String.fromCharCode((c3 & 0x03) << 6 | c4);
    }
    return out;
};    

/**
 * Base64 encodes a string, even binary ones. Succeeds where
 * window.btoa() fails.
 *
 * @param  String  str  The string content to encode
 * @return string
 */
var encode = function encode(str) {
    /*eslint max-statements:0 */
    var out = "", i = 0, len = str.length, c1, c2, c3;
    while (i < len) {
        c1 = str.charCodeAt(i++) & 0xff;
        if (i === len) {
            out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
            out += BASE64_ENCODE_CHARS.charAt((c1 & 0x3) << 4);
            out += "==";
            break;
        }
        c2 = str.charCodeAt(i++);
        if (i === len) {
            out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
            out += BASE64_ENCODE_CHARS.charAt((c1 & 0x3) << 4 | (c2 & 0xF0) >> 4);
            out += BASE64_ENCODE_CHARS.charAt((c2 & 0xF) << 2);
            out += "=";
            break;
        }
        c3 = str.charCodeAt(i++);
        out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
        out += BASE64_ENCODE_CHARS.charAt((c1 & 0x3) << 4 | (c2 & 0xF0) >> 4);
        out += BASE64_ENCODE_CHARS.charAt((c2 & 0xF) << 2 | (c3 & 0xC0) >> 6);
        out += BASE64_ENCODE_CHARS.charAt(c3 & 0x3F);
    }
    return out;
};