how to remove completed jobs older than x from kue

Question

I'm using kue for node.js and I see there is sample code for removing jobs on complete, but is there a way I can remove stale jobs older than X? I'd like to see completed jobs for a day or two so that I can review what's been happening, but for it to clean up anything older than that.

Answer 1

The kue API appears to have improved considerably since this question was first asked. I dug into the code a bit and this much simpler version worked for me:

var kue = require('kue');
kue.Job.rangeByState('complete', 0, someLargeNumber, 1, function(err, jobs) {
  jobs.forEach(function(job) {
    if (job.created_at < yourTimeThreshold) return;
    job.remove();  
  });
});

(Error handling is omitted for brevity's sake.)

Answer 2

It would be nice if Kue provided a mechanism to clean-up completed jobs within the library, however you can implement a solution based on each jobs 'created_at' field. Here is what I use to remove completed jobs older than 7 days, scanning every 24 hours.

    var _ = require('underscore'),
        kue = require('kue');


    var jobs = kue.createQueue(),
        q = new kue, // object so we can access exposed methods in the kue lib
        hours = 24,
        timer = hours * 60 * 60 * 1000; // timer for the setInterval function


    var completedJobs = function(callback) {
      /**
       *  completedJobs - uses the kue lib .complete function to get a list of 
       *  all completed job ids, iterates through each id to retrieve the actual
       *  job object, then pushes the object into an array for the callback.
       *  
       */
      q.complete(function(err, ids){
        var jobs = [],
            count = 0,
            total = ids.length;
        console.log('completedJobs -> ids.length:%s',ids.length);
        _.each(ids, function(id){
          kue.Job.get(id, function(err, job){
            count++;
            jobs.push(job);        
            if (total === count) {
              callback(null, jobs);
              return;
            }
          });
        });
      });
    }

    var removeJobs = function(jobs, callback) {
      /**
       *  removeJobs - removes the job from kue by calling the job.remove from the 
       *  job object collected in completedJobs().
       *  
       */
      var count = 0,
          total = jobs.length;
      console.log('removeJobs -> jobs.length:%s',jobs.length);
      _.each(jobs, function(job) {
        job.remove(function(err) {
          count++;
          if (total === count) {
            callback(null, count);
            return;
          }
        });
      });
    }

    var dateDiffInDays = function(d1, d2) {
      /**
       *  dateDiffInDays - returns the difference between two Date objects in days
       */
      var t2 = d2.getTime(),
          t1 = d1.getTime();
      return parseInt((t2-t1)/(24*3600*1000));
    }

    setInterval(function() {
      /**
       *  setInterval - calls completedJobs in a 24-hour interval
       */
      completedJobs(function(err, jobs) {
        // callback to completedJobs
        console.log('completedJobs -> callback-> jobs.length:%s', jobs.length);
        var jobsToRemove = [],      
            now = new Date();
        _.each(jobs, function(job){
          var then = new Date(parseInt(job.created_at)),
              diff = dateDiffInDays(then, now), 
              timePastForRemoval = 7; // remove anything older than 7 days
          if (diff >= timePastForRemoval) {
            jobsToRemove.push(job);
          }
        });
        console.log('completedJobs -> callback -> jobsToRemove.length:%s', jobsToRemove.length);
        if (jobsToRemove.length > 0) { // if we have jobsToRemove
          removeJobs(jobsToRemove, function(err, count){
            // callback to removeJobs
            console.log('removeJobs -> callback -> jobs removed:%s',count);        
          });
        } else {
          console.log('completedJobs -> callback -> no jobs to remove');      
        }
      });
    }, timer);

    console.log('Running kue completed job clean-up');

Answer 3

I had to do this and found the code above a bit hard to follow, I made a easy to extend clear and concise method here: https://gist.github.com/4212207

Answer 4

I wanted to remove stale jobs from Kue as well, and modified Chris's code to work without Underscore.js.

The code is here: https://gist.github.com/niravmehta/6112330

You can run it on command line as

node kue_cleanup 

Thanks Chris for his wonderful script! It was a great starting point.