NodeJS Cluster on AWS Lambda

Question

Is it possible to use the cluster module in lambda functions? I tried this:

'use strict';
var cluster = require('cluster');  
var http    = require('http');  
var os      = require('os');

var numCPUs = os.cpus().length;
console.log('Number of Cores : ', numCPUs);

exports.test = (event, context, callback) => {
    if (cluster.isMaster) {
        for (var i = 0; i < numCPUs; ++i) {
            cluster.fork();
        }
    } else {
        console.log('child process ');
    }
}

number of cores is always 2, but i never see the child process log.

Update for comment example:

I tried implementing the message pattern but i'm still not receiving the message sent by the children. The for loop correctly loops through the cluster workers but never finds a message.

'use strict';
var cluster = require('cluster');  
var http    = require('http');  
var os      = require('os');

var numCPUs = os.cpus().length;
console.log('Number of Cores : ', numCPUs);

exports.test = (event, context, callback) => {
    if (cluster.isMaster) {
        for (var i = 0; i < numCPUs; ++i) {
            cluster.fork();
        }
        for (const id in cluster.workers) {
            cluster.workers[id].on('message', messageHandler);
        }
    } else {
        process.send('running');
    }
};

function messageHandler(msg) {
    console.log(msg);
}

Answer 1

The issue appears to be that there's nothing to invoke the exports.test function for each of the child processes. cluster.fork() invokes a new instance of the file with different parameters that starts from the beginning, , unlike the fork() system call in C which clones the current process and continues from the same line in both parent and child process.

For the parent process, AWS lambda will invoke this function, but the child process just defines the function and then waits.

I'd flip your logic around so that the cluster.isMaster check happens around everything else; I've tested the following locally:

'use strict';
let cluster = require('cluster')
let http = require('http')
let os = require('os')

let numCPUs = os.cpus().length

if (cluster.isMaster) {
  console.log('Number of Cores : ', numCPUs);
  exports.test = (event, context, callback) => {
    for (let i = 0; i < numCPUs; ++i) {
      cluster.fork();
    }
    for (const id in cluster.workers) {
      cluster.workers[id].on('message', messageHandler);
    }
  }
} else {
  process.send('running');
}

function messageHandler(msg) {
  console.log(msg);
}

// The following used for local test

exports.test && exports.test();

Answer 2

Your code looks correct, except you are making the assumption that the call to console.log() from a worker is visible on the console for your master process. This is not always the case, as calls will not propagate up the process tree.

After your calls to fork(), toss this snippet in to receive a message from a child process:

for (var id in cluster.workers) {
  cluster.workers[id].on('message',function () {
    console.log('got message from ' + id + ': ' + msg);
  });
}

Then in your second case of your if/else (where the worker process procures are stored), add this to message to the master:

process.send('running');

You may also send objects (preferable) via process.send(), I just used a String as an example. There is more detailed information on the official docs: Cluster - How it works