How to record a canvas + audio with no new drawing on the canvas

Question

I want to record a video from some script running on an HTML canvas and simultaneously record the audio from the microphone. I am using canvas.captureStream() to create a video stream from the canvas. Then I create an audio stream from getUserMedia() and mix both streams in a single one which is passed to a new MediaRecorder.

The problem is that, by default, no frame is captured in the canvas stream if nothing is painted to the canvas. So the video and audio are desynchronized in the parts where the canvas is inactive.

The code I am using looks like this:

let recording = false;
let mediaRecorder;
let chunks = [];

let newStream, canvasStream


let constrains = {
  audio: {
    channelCount: { ideal: 2, min: 1 },
    sampleRate: 48000,
    sampleSize: 16,
    volume: 1,
    echoCancellation: true,
    noiseSuppression: true,
    autoGainControl: true,
  },
  video: false
};

recButton.addEventListener("click", () => {
  recording = !recording;
  if (recording) {
    recButton.textContent = "Click to Stop";
    recButton.style.color = "red";
    canvasStream = canvas.captureStream(40); // ---> captureStream()


    navigator.mediaDevices.getUserMedia(constrains) // ---> audioStream
      .then(audioStream => {
     
        canvasStream.addTrack(audioStream.getTracks()[0]);   // --> joint the two streams
    

        mediaRecorder = new MediaRecorder(canvasStream, {
          mimeType: 'video/webm; codecs=vp9',
          // ignoreMutedMedia: false
        });

        mediaRecorder.ondataavailable = e => {
          if (e.data.size > 0) {
            chunks.push(e.data);  // chunks = [];   already defined above....
          }
        };
        mediaRecorder.start();
      })
      .catch(err => {
        console.log(err.name, err.message);
      });
  } else {
    recButton.textContent = "Record";
    recButton.style.color = "black";
    mediaRecorder.stop()

    stopTracks(canvasStream);  // function stop defined below
    console.log(mediaRecorder.state)
    setTimeout(() => {                // why this is necessary?
      const blob = new Blob(chunks, {
        type: "video/mp4"
      });
      const url = URL.createObjectURL(blob);
      const a = document.createElement("a");
      a.href = url;
      a.download = "recording.mp4";
      a.click();
      URL.revokeObjectURL(url);
    }, 200);
  }
});

let stopTracks = stream => stream.getTracks().forEach(track => track.stop());

Answer 1

That's a bit of a Chrome bug since even if the video track should indeed have produced no data in this case, the audio track has and the MediaRecorder should have captured it.

The only way to workaround this is to perform a new drawing on the canvas at least in the same painting frame as your call to canvas.captureStream().
With a 2D context, the less intrusive way to do that is to draw a transparent 1x1 rectangle.

let recording = false;
let mediaRecorder;
const chunks = [];

let newStream, canvasStream

const constrains = {
  audio: {
    channelCount: { ideal: 2, min: 1 },
    sampleRate: 48000,
    sampleSize: 16,
    volume: 1,
    echoCancellation: true,
    noiseSuppression: true,
    autoGainControl: true,
  },
  video: false
};

recButton.addEventListener("click", () => {
  recording = !recording;
  if (recording) {
    recButton.textContent = "Click to Stop";
    recButton.style.color = "red";
    canvasStream = canvas.captureStream(40);
    // force activate the Canvas track
    forceEmptyDrawing(canvas);
    navigator.mediaDevices.getUserMedia(constrains) // ---> audioStream
      .then(audioStream => {

        canvasStream.addTrack(audioStream.getTracks()[0]); // --> joint the two streams
        mediaRecorder = new MediaRecorder(canvasStream, {
          // don't forget the audio codec
          mimeType: "video/webm; codecs=vp9,opus",
          // ignoreMutedMedia: false
        });

        mediaRecorder.ondataavailable = e => {
          if (e.data.size > 0) {
            chunks.push(e.data);
          }
        };
        mediaRecorder.start(10);
      })
      .catch(err => {
        console.log(err.name, err.message);
      });
  } else {
    recButton.textContent = "Record";
    recButton.style.color = "black";
    mediaRecorder.stop()
    // wait for the stop event
    // don't rely on some magic timeout
    mediaRecorder.onstop = () => {
      // only when the recorder is completely stopped
      // you can stop the tracks
      stopTracks(canvasStream);
      const blob = new Blob(chunks, {
        type: "video/webm" // you asked for webm, not mp4
      });
      // just to verify we did record something
      console.log(`recorded a ${ blob.size }b file`);
    };
  }
});

const stopTracks = stream => stream.getTracks().forEach(track => track.stop());

// draws a 1x1 transparent rectangle
function forceEmptyDrawing(canvas) {
  const ctx = canvas.getContext("2d");
  const alpha = ctx.globalAlpha;
  ctx.globalAlpha = 0;
  ctx.fillRect(0, 0, 1, 1);
  ctx.globalAlpha = alpha;
}

As a fiddle since StackSnippets can't use gUM (for Chrome only, Firefox and Safari don't support the provided mimeType).