Resuming queued jobs after NodeJS worker restart on Heroku

Question

So I have a fairly simple setup on Heroku. I'm using RabbitMQ for handling background jobs. My setup consists of a node script that runs daily using Heroku Scheduler addon. The scripts adds jobs to the queue, the worker in turn, consumes them and delegates them onto a separate module for handling.

The problem starts after I receive a SIGTERM event that Heroku initiates randomly from time to time, before restarting the instance.

For some reason, after the instance is restarted, the worker is never get back up again. Only when I restart it manually by doing heroku ps:scale worker=0 and heroku ps:scale worker=1 The worker continues to consume the pending jobs.

Here's my worker:

// worker.js
var throng = require('throng');
var jackrabbit = require('jackrabbit');
var logger = require('logfmt');
var syncService = require('./syncService');

var start = function () {
    var queue = jackrabbit(process.env.RABBITMQ_BIGWIG_RX_URL || 'amqp://localhost');

    logger.log({type: 'msg', msg: 'start', service: 'worker'});

     queue
        .default()
        .on('drain', onDrain)
        .queue({name: 'syncUsers'})
        .consume(onMessage)

    function onMessage(data, ack, nack) {

        var promise;
        switch (data.type) {
            case 'updateUser':
                promise = syncService.updateUser(data.target, data.source);
                break;
            case 'createUser':
                promise = syncService.createUser(data.source);
                break;
            case 'deleteUser':
                promise = syncService.deleteUser(data.target);
        }

        promise.then(ack, nack);
    }

    function onDrain() {
        queue.close();
        logger.log({type: 'info', msg: 'sync complete', service:    'worker'});
     }

    process.on('SIGTERM', shutdown);


    function shutdown() {
        logger.log({type: 'info', msg: 'shutting down'});
        queue.close();
        process.exit();
    }

};


throng({
    workers: 1,
    lifetime: Infinity,
    grace: 4000
}, start);

Answer 1

The close() method on the jackrabbit object takes a callback, you should avoid exiting the process until that is finished:

function shutdown() {
    logger.log({type: 'info', msg: 'shutting down'});
    queue.close(function (e) {
      process.exit(e ? 1 : 0);
    });
}