nodejs, redis. check if keys exists and create new if not

Question

I'm new to nodejs and maybe don't got an event system how it should work. Can't find a bug. Please advice. I need a simple task - check for a tag, if it does not exists, set the new key and info about the tag. The problem is - then I run the script for the first time, it's always returns 'keys not exists'. Check the redisdb keys - it creates many tags Here is my code

for (x = 0; x < rows.length; x++) {
    if (rows[x].term_taxonomy_id != 1) {
        var taxonomy = findOne(rterms, rows[x].term_taxonomy_id);
        rc.exists('tag:' + taxonomy.name, function (err, rexists) {
            if (rexists == false) {
                rc.incr('tags:count', function (err, id) {
                    console.log(taxonomy.name+' not exists. result ' + rexists);
                    rc.set('tag:' + taxonomy.name,id);
                    rc.hmset('tag:' + id,
                        'id', id,
                        'title',taxonomy.name,
                        'url', taxonomy.slug
                    );
                });//incr
            }else{
                console.log(taxonomy.name+' exists!'+rexists);
            };
        });//exists
    };//ifrows
});

Here is another one example

var tags = [
  "apple",
  "tiger",
  "mouse",
  "apple",
  "apple",
  "apple",
  "tiger",
  "mouse",
  "mouse",
];
var count =0;
Object.keys(tags).forEach (function (tag) {
  rc.get("tag:"+tags[tag],function(err,rr){
    console.log("get tag "+tags[tag]+" result code "+rr);
    if (rr == null) {
      rc.set("tag:"+tags[tag],"info",function(err,rr){
        count++;
        console.log('set tag '+tags[tag]+' '+rr+' objects count '+count);
      });
    };
  });
})

the output:

get tag apple result code null
get tag tiger result code null
get tag mouse result code null
get tag apple result code null
get tag apple result code null
get tag apple result code null
get tag tiger result code null
get tag mouse result code null
get tag mouse result code null
set tag apple OK objects count 1
set tag tiger OK objects count 2
set tag mouse OK objects count 3
set tag apple OK objects count 4
set tag apple OK objects count 5
set tag apple OK objects count 6
set tag tiger OK objects count 7
set tag mouse OK objects count 8
set tag mouse OK objects count 9

Looks like nodejs executes all 'get' commands and only after that 'set' commands. So... I understand, it's all because of async operations. But how to make it work?

Answer 1

There are at least two problems in this code:

• the first one is linked to Javascript closure management. The body of a loop does not create a scope. With Javascript, scope of variables is at function level, not block level. You need to introduce some function in the loop itself to enforce the creation of a proper closure. More information here.

• the second one is a race condition between the exists and set commands. If you have several Redis connections running exists and set commands on the same keys, you will likely have some kind of conflicts. Instead of using exists and set, you should use setnx which perform the check and set in one atomic operation.

Considering your second example, the closure problem has been fixed by using forEach, but you still generate all the get operations before the set operations due to the asynchronous nature of the language.

If you really want to sequence all your get and set operations (which will be much slower btw), then you can use a bit of functional programming to implement the loop using recursion.

Example:

This program:

var redis = require('redis')
var rc = redis.createClient(6379, 'localhost');

var tags = [
  "apple",
  "tiger",
  "mouse",
  "apple",
  "apple",
  "apple",
  "tiger",
  "mouse",
  "mouse",
];

var count = 0;

function loop(tags) {
  function rec_loop(tags,i) {
     if ( i >= tags.length )
        return
     rc.get("tag:"+tags[i],function(err,rr) {
        console.log("get tag "+tags[i]+" result code "+rr);
        if ( rr == null ) {
           rc.set("tag:"+tags[i],"info",function(err,rr) {
              count++;
              console.log('set tag '+tags[i]+' '+rr+' objects count '+count);
              rec_loop(tags,++i)
           })
        } else
          rec_loop(tags,++i)
     })
  }
  rec_loop(tags,0)
}

loop(tags)

displays:

get tag apple result code null
set tag apple OK objects count 1
get tag tiger result code null
set tag tiger OK objects count 2
get tag mouse result code null
set tag mouse OK objects count 3
get tag apple result code info
get tag apple result code info
get tag apple result code info
get tag tiger result code info
get tag mouse result code info
get tag mouse result code info

Note that the race condition is still present in this example. You are supposed to use setnx to implement this kind of check and set operations.