How to accurately filter RGB value for chroma-key effect

Question

I just read this tutorial and tried this example. So I downloaded a video from web for my own testing. All I have to do is tweak rgb values in if conditions

HERE is the sample code from example

computeFrame: function() {
    this.ctx1.drawImage(this.video, 0, 0, this.width, this.height);
    let frame = this.ctx1.getImageData(0, 0, this.width, this.height);
    let l = frame.data.length / 4;

    for (let i = 0; i < l; i++) {
      let r = frame.data[i * 4 + 0];
      let g = frame.data[i * 4 + 1];
      let b = frame.data[i * 4 + 2];
      if (g > 100 && r > 100 && b < 43)
        frame.data[i * 4 + 3] = 0;
    }
    this.ctx2.putImageData(frame, 0, 0);
    return;
  }

In the tutorial example its filtering out yellow(not yellow I guess) color. The sample video I downloaded uses green background. So I tweaked rgb value in if condition to get desired results

After multiple tries, I managed to get this.

Now what I want to know is how can I accurately filter out green screen (or any other screen)perfectly without guessing. Or randomly tweaking values.

With just guessing it take hours to get it perfectly right. And this is just a sample with a real world application. It can take maybe more.

NOTE: The example is working in Firefox for now..

Answer 1

You probably just need a better algorithm. Here's one, it's not perfect, but you can tweak it a lot easier.

Basically you'll just need a colorpicker, and pick the lightest and darkest values from the video (putting the RGB values in the l_ and d_ variables respectively). You can adjust the tolerance a little bit if you need to, but getting the l_ and r_ values just right by picking different areas with the color picker will give you a better key.

let l_r = 131,
    l_g = 190,
    l_b = 137,

    d_r = 74,
    d_g = 148,
    d_b = 100;

let tolerance = 0.05;

let processor = {
  timerCallback: function() {
    if (this.video.paused || this.video.ended) {
      return;
    }
    this.computeFrame();
    let self = this;
    setTimeout(function () {
        self.timerCallback();
      }, 0);
  },

  doLoad: function() {
    this.video = document.getElementById("video");
    this.c1 = document.getElementById("c1");
    this.ctx1 = this.c1.getContext("2d");
    this.c2 = document.getElementById("c2");
    this.ctx2 = this.c2.getContext("2d");
    let self = this;
    this.video.addEventListener("play", function() {
        self.width = self.video.videoWidth;
        self.height = self.video.videoHeight;
        self.timerCallback();
      }, false);
  },

  calculateDistance: function(c, min, max) {
      if(c < min) return min - c;
      if(c > max) return c - max;

      return 0;
  },

  computeFrame: function() {
    this.ctx1.drawImage(this.video, 0, 0, this.width, this.height);
    let frame = this.ctx1.getImageData(0, 0, this.width, this.height);
        let l = frame.data.length / 4;

    for (let i = 0; i < l; i++) {
      let _r = frame.data[i * 4 + 0];
      let _g = frame.data[i * 4 + 1];
      let _b = frame.data[i * 4 + 2];

      let difference = this.calculateDistance(_r, d_r, l_r) + 
                       this.calculateDistance(_g, d_g, l_g) +
                       this.calculateDistance(_b, d_b, l_b);
      difference /= (255 * 3); // convert to percent
      if (difference < tolerance)
        frame.data[i * 4 + 3] = 0;
    }
    this.ctx2.putImageData(frame, 0, 0);
    return;
  }
};
// :/ 

Answer 2

If performance does not matter, then you could work in another color space e.g. HSV. You could use the left top pixel as reference.

You compare the hue value of the reference point with hue value other pixels, and exclude all pixels that exceed a certain threshold and dark and light areas using saturation and value.

This how ever does not completely get rid of color bleeding, there you might need to do some color correct/desaturation.

function rgb2hsv () {
    var rr, gg, bb,
        r = arguments[0] / 255,
        g = arguments[1] / 255,
        b = arguments[2] / 255,
        h, s,
        v = Math.max(r, g, b),
        diff = v - Math.min(r, g, b),
        diffc = function(c){
            return (v - c) / 6 / diff + 1 / 2;
        };

    if (diff == 0) {
        h = s = 0;
    } else {
        s = diff / v;
        rr = diffc(r);
        gg = diffc(g);
        bb = diffc(b);

        if (r === v) {
            h = bb - gg;
        }else if (g === v) {
            h = (1 / 3) + rr - bb;
        }else if (b === v) {
            h = (2 / 3) + gg - rr;
        }
        if (h < 0) {
            h += 1;
        }else if (h > 1) {
            h -= 1;
        }
    }
    return {
        h: Math.round(h * 360),
        s: Math.round(s * 100),
        v: Math.round(v * 100)
    };
}


let processor = {
  timerCallback: function() {
    if (this.video.paused || this.video.ended) {
      return;
    }
    this.computeFrame();
    let self = this;
    setTimeout(function () {
        self.timerCallback();
      }, 0);
  },

  doLoad: function() {
    this.video = document.getElementById("video");
    this.c1 = document.getElementById("c1");
    this.ctx1 = this.c1.getContext("2d");
    this.c2 = document.getElementById("c2");
    this.ctx2 = this.c2.getContext("2d");
    let self = this;
    this.video.addEventListener("play", function() {
        self.width = self.video.videoWidth / 2;
        self.height = self.video.videoHeight / 2;
        self.timerCallback();
      }, false);
  },

  computeFrame: function() {
    this.ctx1.drawImage(this.video, 0, 0, this.width, this.height);
    let frame = this.ctx1.getImageData(0, 0, this.width, this.height);
        let l = frame.data.length / 4;


    let reference = rgb2hsv(frame.data[0], frame.data[1], frame.data[2]);

    for (let i = 0; i < l; i++) {
      let r = frame.data[i * 4 + 0];
      let g = frame.data[i * 4 + 1];
      let b = frame.data[i * 4 + 2];
      let hsv = rgb2hsv(r, g, b);

      let hueDifference = Math.abs(hsv.h - reference.h);

      if( hueDifference < 20 && hsv.v > 50 && hsv.s > 50 ) {
        frame.data[i * 4 + 3] = 0;
      }


    }
    this.ctx2.putImageData(frame, 0, 0);
    return;
  }
};