Dynamically Running Mocha Tests

Question

I'm trying to run a series of tests dynamically. I have the following setup but it doesn't seem to run and I'm not getting any errors:

import Mocha from 'mocha';
const Test = Mocha.Test;
const Suite = Mocha.Suite;
const mocha = new Mocha();
for (let s in tests) {
  let suite = Suite.create(mocha.suite, s);
  tests[s].forEach((test) => {
    console.log('add test', test.name)
    suite.addTest(new Test(test.name), () => {
      expect(1+1).to.equal(2);
    });
  });
}
mocha.run();

The tests I'm running look like this:

{ todo: 
  [ { name: 'POST /todos',
      should: 'create a new todo',
      method: 'POST',
      endpoint: '/todos',
      body: [Object] } ] }

(though at this point my test is just trying to check a basic expect)

Based on the console.logs the iteration seems fine and it appears to be adding the tests, so I'm confident in the flow of operations, I just can't get any execution or errors.

Answer 1

You have to pass the test function to the Test constructor, not to suite.addTest. So change your code to add your tests like this:

suite.addTest(new Test(test.name, () => {
    expect(1+1).to.equal(2);
}));

Here is the entire code I'm running, adapted from your question:

import Mocha from 'mocha';
import { expect } from 'chai';
const Test = Mocha.Test;
const Suite = Mocha.Suite;
const mocha = new Mocha();

var tests = { todo:
  [ { name: 'POST /todos',
      should: 'create a new todo',
      method: 'POST',
      endpoint: '/todos',
      body: [Object] } ] };

for (let s in tests) {
  let suite = Suite.create(mocha.suite, s);
  tests[s].forEach((test) => {
      console.log('add test', test.name);
      suite.addTest(new Test(test.name, () => {
          expect(1+1).to.equal(2);
      }));
  });
}
mocha.run();

When I run the above with node_modules/.bin/babel-node test.es6, I get the output:

  todo
    ✓ POST /todos


  1 passing (5ms)

Answer 2

It's critical to test your test system and make sure it deals with passing and failing tests and thrown exceptions. Since folks are counting on a build process to warn them about errors, you must also set the exit code to a non-zero if anything failed. Below is a test script (which you must invoke with node test.js rather than mocha test.js) which exercises all paths through your test suite:

const Mocha = require('mocha')
const expect = require('chai').expect
var testRunner = new Mocha()
var testSuite = Mocha.Suite.create(testRunner.suite, 'Dynamic tests')

var tests = [ // Define some tasks to add to test suite.
  { name: 'POST /todos', f: () => true }, //              Pass a test.
  { name: 'GET /nonos',  f: () => false }, //             Fail a test.
  { name: 'HEAD /hahas', f: () => { throw Error(0) } } // Throw an error.
]

tests.forEach(
  test =>
    // Create a test which value errors and caught exceptions.
    testSuite.addTest(new Mocha.Test(test.name, function () {
      expect(test.f()).to.be.true
    }))
)
var suiteRun = testRunner.run() //             Run the tests
process.on('exit', (code) => { //              and set exit code.
  process.exit(suiteRun.stats.failures > 0) // Non-zero exit indicates errors.
}) // Falling off end waits for Mocha events to finish.

---

Given that this is prominent in web searches for asynchronous mocha tests, I'll provide a couple more useful templates for folks to copy.

Embedded execution: The first directly adds tests which invoke an asynchronous faux-network call and check the result in a .then:

const Mocha = require('mocha')
const expect = require('chai').expect
var testRunner = new Mocha()
var testSuite = Mocha.Suite.create(testRunner.suite, 'Network tests')

var tests = [ // Define some long async tasks.
  { name: 'POST /todos', pass: true, wait: 3500, exception: null },
  { name: 'GET /nonos', pass: false, wait: 2500, exception: null },
  { name: 'HEAD /hahas', pass: true, wait: 1500, exception: 'no route to host' }
]

tests.forEach(
  test =>
    // Create a test which value errors and caught exceptions.
    testSuite.addTest(new Mocha.Test(test.name, function () {
      this.timeout(test.wait + 100) // so we can set waits above 2000ms
      return asynchStuff(test).then(asyncResult => {
        expect(asyncResult.pass).to.be.true
      }) // No .catch() needed because Mocha.Test() handles them.
    }))
)
var suiteRun = testRunner.run() //             Run the tests
process.on('exit', (code) => { //              and set exit code.
  process.exit(suiteRun.stats.failures > 0) // Non-zero exit indicates errors.
}) // Falling off end waits for Mocha events to finish.

function asynchStuff (test) {
  return new Promise(function(resolve, reject) {
    setTimeout(() => {
//    console.log(test.name + ' on ' + test.endpoint + ': ' + test.wait + 'ms')
      if (test.exception)
        reject(Error(test.exception))
      resolve({name: test.name, pass: test.pass}) // only need name and pass
    }, test.wait)
  })
}

This code handles passing and failing data, reports exceptions, and exits with a non-zero status if there were errors. The output reports all expected problems and additionally whines about the test taking a like time (3.5s):

  Network tests
    ✓ POST /todos (3504ms)
    1) GET /nonos
    2) HEAD /hahas
  1 passing (8s)
  2 failing

  1) Network tests GET /nonos:
      AssertionError: expected false to be true
      + expected - actual    
      -false
      +true

  2) Network tests HEAD /hahas:
     Error: no route to host

---

Delayed execution: This approach invokes all of the slow tasks before populating and starting the the mocha test suite:

const Mocha = require('mocha')
const expect = require('chai').expect
var testRunner = new Mocha()
var testSuite = Mocha.Suite.create(testRunner.suite, 'Network tests')

var tests = [ // Define some long async tasks.
  { name: 'POST /todos', pass: true, wait: 3500, exception: null },
  { name: 'GET /nonos', pass: false, wait: 2500, exception: null },
  { name: 'HEAD /hahas', pass: true, wait: 1500, exception: 'no route to host' }
]

Promise.all(tests.map( // Wait for all async operations to finish.
  test => asynchStuff(test)
    .catch(e => { // Resolve caught errors so Promise.all() finishes.
      return {name: test.name, caughtError: e}
    })
)).then(testList => // When all are done,
  testList.map( //     for each result,
    asyncResult => //  test value errors and exceptions.
      testSuite.addTest(new Mocha.Test(asyncResult.name, function () {
        if (asyncResult.caughtError) { // Check test object for caught errors
          throw asyncResult.caughtError
        }
        expect(asyncResult.pass).to.be.true
      }))
  )
).then(x => { //                                 When all tests are created,
  var suiteRun = testRunner.run() //             run the tests
  process.on('exit', (code) => { //              and set exit code.
    process.exit(suiteRun.stats.failures > 0) // Non-zero exit indicates errors.
  })
})

function asynchStuff (test) {
  return new Promise(function(resolve, reject) {
    setTimeout(() => {
//    console.log(test.name + ' on ' + test.endpoint + ': ' + test.wait + 'ms')
      if (test.exception)
        reject(Error(test.exception))
      resolve({name: test.name, pass: test.pass}) // only need name and pass
    }, test.wait)
  })
}

The output is the same except that mocha doesn't whine about the slow test and instead believes the tests tool less than 10ms. The Promise.all waits for all the promises to resolve or reject then creates the tests to validate the results or report exceptions. This is a few lines longer than Embedded execution because it must:

1. Resolve exceptions so Promise.all() resolves.
2. Execute the tests in a final Promise.all().then()

Comments describing how folks pick which style to use could guide others. Share your wisdom!