karma-sauce-launcher disconnects every test run resulting in failed runs with IE and Safari

Question

I have been having this exact issue for months now, through multiple versions of each component involved (karma, socket.io, sauce-connect). Currently I'm running on Karma 12.16.

Basically, when using https://github.com/karma-runner/karma-sauce-launcher, Safari and IE browsers launched disconnect every single test run resulting in only a partial test run. The relevant Karma log is

Driving the web on session: abf7c976abf1420a9c72b3e06c655040    
DEBUG [wd]:  > RESPONSE init({"base":"SauceLabs","browserName":"internet explorer","platform":"Windows 8.1","version":"11","tags":[],"name":"Karma and Sauce Labs demo","record-video":false,"record-screenshots":true,"build":null,"device-orientation":null,"disable-popup-handler":true}) "abf7c976abf1420a9c72b3e06c655040",null
INFO [launcher.sauce]: internet explorer 11 (Windows 8.1) session at https://saucelabs.com/tests/abf7c976abf1420a9c72b3e06c655040
DEBUG [launcher.sauce]: WebDriver channel for internet explorer 11 (Windows 8.1) instantiated, opening http://localhost:9876/?id=15483642
DEBUG [wd]:  > CALL get("http://localhost:9876/?id=15483642") 
DEBUG [wd]:  > POST /session/:sessionID/url {"url":"http://localhost:9876/?id=15483642"}
DEBUG [web-server]: serving: /Volumes/OSXSTORAGE/Users/a/Dropbox/a/WebstormProjects/a-ui/node_modules/karma/static/client.html
DEBUG [web-server]: serving: /Volumes/OSXSTORAGE/Users/a/Dropbox/a/WebstormProjects/a-ui/node_modules/karma/static/karma.js
DEBUG [wd]:  > RESPONSE get("http://localhost:9876/?id=15483642") 
DEBUG [karma]: A browser has connected on socket xXySXoeppVtPhAr8FVR4
INFO [IE 11.0.0 (Windows)]: Connected on socket xXySXoeppVtPhAr8FVR4 with id 15483642
DEBUG [launcher]: internet explorer 11 (Windows 8.1) on SauceLabs (id 15483642) captured in 17.693 secs
DEBUG [web-server]: serving: /Volumes/OSXSTORAGE/Users/a/Dropbox/a/WebstormProjects/a-ui/node_modules/karma/static/context.html
DEBUG [web-server]: serving (cached): /Volumes/OSXSTORAGE/Users/a/Dropbox/a/WebstormProjects/a-ui/src/components/angular/angular.js
................................................................................
..............DEBUG [karma]: A browser has connected on socket mGf0VzQ8DKdfW_NSFVR5
DEBUG [IE 11.0.0 (Windows)]: New connection mGf0VzQ8DKdfW_NSFVR5 (already have xXySXoeppVtPhAr8FVR4)
WARN [IE 11.0.0 (Windows)]: Disconnected (1 times), because no message in 10000 ms.

IE 11.0.0 (Windows): Executed 94 of 94 DISCONNECTED (11.481 secs / 0.757 secs)
DEBUG [launcher.sauce]: Shutting down the internet explorer 11 (Windows 8.1) driver
DEBUG [wd]:  > CALL get("about:blank") 
DEBUG [wd]:  > POST /session/:sessionID/url {"url":"about:blank"}
DEBUG [karma]: Run complete, exitting.
DEBUG [launcher]: Disconnecting all browsers
DEBUG [wd]:  > RESPONSE get("about:blank") 
DEBUG [wd]:  > CALL quit() 
DEBUG [wd]:  > DELETE /session/:sessionID 
DEBUG [wd]: 
Ending your web drivage..

DEBUG [wd]:  > RESPONSE quit() 

I have tried increasing browserNoActivityTimeout to 60000, but that just results in the tests failing after 60s instead of 10s. Every one of these failed has the following:

DEBUG [karma]: A browser has connected on socket mGf0VzQ8DKdfW_NSFVR5
DEBUG [IE 11.0.0 (Windows)]: New connection mGf0VzQ8DKdfW_NSFVR5 (already have xXySXoeppVtPhAr8FVR4)
WARN [IE 11.0.0 (Windows)]: Disconnected (1 times), because no message in 10000 ms.   
IE 11.0.0 (Windows): Executed 94 of 94 DISCONNECTED (11.481 secs / 0.757 secs)

Depending on the browser, sometimes it will only execute 50 tests (Executed 50 of 94 DISCONNECTED), and on others it will execute all tests, but still be considered a failed test run. I have never seen this occur with Chrome or Firefox.

The exact same karma.conf file will run every time locally with no issues. Has anyone encountered this? I have reached out to saucelabs support, but they are starting to insist that this is a karma-sauce-launcher issue.

Karma.conf:

var fs = require('fs');

module.exports = function(config) {

  // Use ENV vars on Travis and sauce.json locally to get credentials
  if (!process.env.SAUCE_USERNAME) {
    if (!fs.existsSync('sauce.json')) {
      console.log('Create a sauce.json with your credentials based on the sauce-sample.json file.');
      process.exit(1);
    } else {
      process.env.SAUCE_USERNAME = require('./../sauce').username;
      process.env.SAUCE_ACCESS_KEY = require('./../sauce').accessKey;
    }
  }

  // Browsers to run on Sauce Labs
  var customLaunchers = {

    sl_safari_ml: {
      base: 'SauceLabs',
      browserName: 'safari',
      platform: 'OS X 10.9'

    }
  };

  config.set({

    // base path that will be used to resolve all patterns (eg. files, exclude)
    basePath: '../',


    // frameworks to use
    // available frameworks: https://npmjs.org/browse/keyword/karma-adapter
    frameworks: ['jasmine'],


    // list of files / patterns to load in the browser
    files: [
      'src/components/jquery/dist/jquery.js',
      'src/components/angular/angular.js',
      'src/components/angular-mocks/angular-mocks.js',
      'src/components/angular-cookies/angular-cookies.js',
      'src/components/angular-sanitize/angular-sanitize.js',
      'src/components/angular-ui-router/release/angular-ui-router.js',
      'src/components/angular-animate/angular-animate.js',
      'src/components/angular-strap/dist/angular-strap.js',
      'src/components/angular-strap/dist/angular-strap.tpl.js',
      'src/components/speakingurl/lib/index.js',
      'src/app/**/*.js',
      'src/app/common/templates/*.html'


    ],

    exclude: [
      'src/app/**/*.e2e.spec.js'

    ],
    ngHtml2JsPreprocessor: {
      stripPrefix: 'src/app/common/',
      moduleName: 'mywire.templates'
    },
    preprocessors: {
      // load templates
      'src/app/common/templates/*.html': ['ng-html2js']
    },
    // test results reporter to use
    // possible values: 'dots', 'progress'
    // available reporters: https://npmjs.org/browse/keyword/karma-reporter
    reporters: ['dots'],


    // web server port
    port: 9876,

    colors: true,

    // level of logging
    // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
    logLevel: config.LOG_INFO,

    sauceLabs: {
      testName: 'Karma and Sauce Labs demo',
      startConnect: false
    },
    browserNoActivityTimeout: 60000,
    captureTimeout: 120000,
    customLaunchers: customLaunchers,

    // start these browsers
    // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
    browsers: Object.keys(customLaunchers),
    singleRun: true
  });
};

Answer 1

Starting Karma version 0.12 there are some more timeout settings added.

Adding them to my karma config resolved this issue for me.

browserDisconnectTimeout

• Type: Number
• Default: 2000
• Description: How long does Karma wait for a browser to reconnect (in ms). With a flaky connection it is pretty common that browser disconnects but the actual execution is running without any problems. Karma does not treat a disconnection as immediate failure and rather wait browserDisconnectTimeout ms. If the browser reconnect during that time, everything is fine.

-

browserDisconnectTolerance

• Type: Number
• Default: 0
• Description: The number of disconnections tolerated. The disconnectTolerance value represents the maximum number of tries a browser will attempt in case of disconnection. Usually any disconnection is considered as a failure, but this option allows to define a tolerance level when there is a flaky network link between the karma server and the browsers.

-

browserNoActivityTimeout

• Type: Number
• Default: 10000
• Description: How long does Karma wait for a message from a browser before disconnecting it (in ms). If, during the execution, Karma does not receive any message from a browser within browserNoActivityTimeout ms, it will disconnect the browser.

source: http://karma-runner.github.io/0.12/config/configuration-file.html

more tips: http://oligofren.wordpress.com/2014/05/27/running-karma-tests-on-browserstack/